Commit 37db6a76 authored by Willem ter Berg's avatar Willem ter Berg

Merge public.version changes to data.overheid.nl

parents 51f0d4df dea308f6
......@@ -18,23 +18,23 @@ logging.basicConfig(
)
_VALUELISTS = {
'adms_changetype': 'http://waardelijsten.dcat-ap-donl.nl/adms_changetype.json',
'adms_distributiestatus': 'http://waardelijsten.dcat-ap-donl.nl/adms_distributiestatus.json',
'donl_catalogs': 'http://waardelijsten.dcat-ap-donl.nl/donl_catalogs.json',
'donl_language': 'http://waardelijsten.dcat-ap-donl.nl/donl_language.json',
'donl_organization': 'http://waardelijsten.dcat-ap-donl.nl/donl_organization.json',
'iana_mediatypes': 'http://waardelijsten.dcat-ap-donl.nl/iana_mediatypes.json',
'mdr_filetype_nal': 'http://waardelijsten.dcat-ap-donl.nl/mdr_filetype_nal.json',
'overheid_dataset_status': 'http://waardelijsten.dcat-ap-donl.nl/overheid_dataset_status.json',
'overheid_frequency': 'http://waardelijsten.dcat-ap-donl.nl/overheid_frequency.json',
#'overheid_license': 'http://waardelijsten.dcat-ap-donl.nl/overheid_license.json',
'overheid_openbaarheidsniveau': 'http://waardelijsten.dcat-ap-donl.nl/overheid_openbaarheidsniveau.json',
'overheid_spatial_scheme': 'http://waardelijsten.dcat-ap-donl.nl/overheid_spatial_scheme.json',
'overheid_spatial_gemeente': 'http://waardelijsten.dcat-ap-donl.nl/overheid_spatial_gemeente.json',
'overheid_spatial_koninkrijksdeel': 'http://waardelijsten.dcat-ap-donl.nl/overheid_spatial_koninkrijksdeel.json',
'overheid_spatial_provincie': 'http://waardelijsten.dcat-ap-donl.nl/overheid_spatial_provincie.json',
'overheid_spatial_waterschap': 'http://waardelijsten.dcat-ap-donl.nl/overheid_spatial_waterschap.json',
'overheid_taxonomiebeleidsagenda': 'http://waardelijsten.dcat-ap-donl.nl/overheid_taxonomiebeleidsagenda.json'
'adms_changetype': 'https://waardelijsten.dcat-ap-donl.nl/adms_changetype.json',
'adms_distributiestatus': 'https://waardelijsten.dcat-ap-donl.nl/adms_distributiestatus.json',
'donl_catalogs': 'https://waardelijsten.dcat-ap-donl.nl/donl_catalogs.json',
'donl_language': 'https://waardelijsten.dcat-ap-donl.nl/donl_language.json',
'donl_organization': 'https://waardelijsten.dcat-ap-donl.nl/donl_organization.json',
'iana_mediatypes': 'https://waardelijsten.dcat-ap-donl.nl/iana_mediatypes.json',
'mdr_filetype_nal': 'https://waardelijsten.dcat-ap-donl.nl/mdr_filetype_nal.json',
'overheid_dataset_status': 'https://waardelijsten.dcat-ap-donl.nl/overheid_dataset_status.json',
'overheid_frequency': 'https://waardelijsten.dcat-ap-donl.nl/overheid_frequency.json',
#'overheid_license': 'https://waardelijsten.dcat-ap-donl.nl/overheid_license.json',
'overheid_openbaarheidsniveau': 'https://waardelijsten.dcat-ap-donl.nl/overheid_openbaarheidsniveau.json',
'overheid_spatial_scheme': 'https://waardelijsten.dcat-ap-donl.nl/overheid_spatial_scheme.json',
'overheid_spatial_gemeente': 'https://waardelijsten.dcat-ap-donl.nl/overheid_spatial_gemeente.json',
'overheid_spatial_koninkrijksdeel': 'https://waardelijsten.dcat-ap-donl.nl/overheid_spatial_koninkrijksdeel.json',
'overheid_spatial_provincie': 'https://waardelijsten.dcat-ap-donl.nl/overheid_spatial_provincie.json',
'overheid_spatial_waterschap': 'https://waardelijsten.dcat-ap-donl.nl/overheid_spatial_waterschap.json',
'overheid_taxonomiebeleidsagenda': 'https://waardelijsten.dcat-ap-donl.nl/overheid_taxonomiebeleidsagenda.json'
}
# We skip overheid_license since it requires a full CKAN restart for changes in that file to
# take effect.
......@@ -82,6 +82,6 @@ if __name__ == '__main__':
for key, value in _VALUELISTS.items():
logging.info('Updating [%s] from [%s]', key, value)
update_valuelist(key, value)
sleep(3)
sleep(2)
logging.info('ValuelistUpdater finished.')
......@@ -46,6 +46,7 @@ def create_schema(schema):
contact_point_validation = tk.get_validator('contact_point_validation')
hash_validation = tk.get_validator('hash_validation')
legal_foundation_validation = tk.get_validator('legal_foundation_validation')
rights_validation = tk.get_validator('rights_validation')
to_curly_string_if_list = tk.get_converter('convert_to_curly_string_if_list')
schema.update({
......@@ -96,7 +97,8 @@ def create_schema(schema):
temporal_validation,
contact_point_validation,
legal_foundation_validation,
date_planned_validation]
date_planned_validation,
rights_validation]
})
schema['resources'].update({
......@@ -165,6 +167,7 @@ def update_schema(schema):
contact_point_validation = tk.get_validator('contact_point_validation')
hash_validation = tk.get_validator('hash_validation')
legal_foundation_validation = tk.get_validator('legal_foundation_validation')
rights_validation = tk.get_validator('rights_validation')
schema.update({
'identifier': [mandatory, is_string, to_extras],
......@@ -215,7 +218,8 @@ def update_schema(schema):
temporal_validation,
contact_point_validation,
legal_foundation_validation,
date_planned_validation]
date_planned_validation,
rights_validation]
})
schema['resources'].update({
......
......@@ -22,7 +22,6 @@ def mock_ckan():
sys.modules['ckan.plugins'] = CkanPlugins
sys.modules['ckan.plugins.toolkit'] = CkanPluginsToolkit
def run_tests():
from validator.test_changetype_validator import TestChangetypeValidator
from validator.test_contact_point_validator import TestContactPointValidator
......@@ -47,7 +46,8 @@ def run_tests():
}
for name, testcases in tests.iteritems():
print '----------------------------------------------------------------------\n ' + name
print '----------------------------------------------------------------------\n '\
+ name
unittest.TextTestRunner(verbosity=0).run(testcases)
......
# encoding: utf-8
"""
Provides complex rights validation for a DCAT-AP-DONL dataset.
"""
def validate_stated_rights(key, data_dict, errors, context):
"""
Validates that the various rights statements do not conflict each other.
:param key: str, argument to match CKAN method signature, not used
:param data_dict: dict, flattened dict of the dataset
:param errors: dict, flattened dict of the validation errors
:param context: dict, argument to match CKAN method signature, not used
:return: (key, data, errors, context), the original arguments, possibly modified
"""
non_open_licenses = [
{'id': 'http://standaarden.overheid.nl/owms/terms/geslotenlicentie'},
{'id': 'http://standaarden.overheid.nl/owms/terms/licentieonbekend'}
]
status = data_dict.get(('dataset_status',), None)
access_rights = data_dict.get(('access_rights',), None)
license = data_dict.get(('license',), None)
status_available = status == 'http://data.overheid.nl/status/beschikbaar'
status_closed = status == 'http://data.overheid.nl/status/niet_beschikbaar'
access_rights_public = access_rights == 'http://publications.europa.eu/resource/authority/access-right/PUBLIC'
access_rights_restricted = access_rights == 'http://publications.europa.eu/resource/authority/access-right/RESTRICTED'
license_is_open = license not in non_open_licenses
if status_available and access_rights_public and not license_is_open:
errors[('license',)] = ['An open license must be provided when dataset_status is '
'[http://data.overheid.nl/status/beschikbaar] and access_rights '
'is [http://publications.europa.eu/resource/authority/access-right/PUBLIC]']
if status_available and not license_is_open and not access_rights_restricted:
errors[('access_rights',)] = ['access_rights must be '
'[http://publications.europa.eu/resource/authority/access-right/RESTRICTED] '
'when status is [http://data.overheid.nl/status/beschikbaar] '
'and a non-open license is provided']
if status_closed and license_is_open:
errors[('license',)] = ['cannot provide an open license when dataset_status is '
'[http://data.overheid.nl/status/niet_beschikbaar]']
return key, data_dict, errors, context
......@@ -64,19 +64,18 @@ def valid_spatial(key, data, errors, context):
schemes = data[('spatial_scheme',)]
values = data[('spatial_value',)]
for val in values:
is_valid = False
if len(schemes) != len(values):
errors[('spatial_value',)].append('The amount of spatial_schemes does not correspond with '
'the amount of spatial_values')
errors[('spatial_scheme',)].append('The amount of spatial_schemes does not correspond with '
'the amount of spatial_values')
for scheme in schemes:
validator_name = _get_validator_for_spatial_scheme(scheme)
validates = validator_factory.get_validator(validator_name)
if validates(val):
is_valid = True
return key, data, errors, context
if not is_valid:
for iterator, val in enumerate(values):
if not validator_factory.get_validator(_get_validator_for_spatial_scheme(schemes[iterator]))(val):
errors[('spatial_value',)].append('value [' + val + '] is not a valid spatial according'
' to the schemes provided')
' to the scheme provided')
return key, data, errors, context
......
......@@ -87,6 +87,6 @@ def date_planned_validation(key, data_dict, errors, context):
if data_dict[dataset_status_tuple] in affected_states:
if not ('date_planned',) in data_dict:
errors[('date_planned',)] = ['date_planned is mandatory when dataset_status is gepland '
'or in onderzoek']
'or in_onderzoek']
return key, data_dict, errors, context
......@@ -13,6 +13,7 @@ import ckanext.dcatdonl.validator.hash_validator as hash_validator
import ckanext.dcatdonl.validator.legal_foundation_validator as legal_foundation_validator
import ckanext.dcatdonl.validator.valuelist_validator as valuelist_validator
import ckanext.dcatdonl.validator.changetype_validator as changetype_validator
import ckanext.dcatdonl.validator.rights_validator as rights_validator
class ValidatorException(Exception):
......@@ -34,6 +35,7 @@ _VALIDATORS = {
'contact_point_validation': contact_point_validator.valid_contact_point,
'hash_validation': hash_validator.valid_hash,
'legal_foundation_validation': legal_foundation_validator.valid_legal_foundation,
'rights_validation': rights_validator.validate_stated_rights,
'overheid_dataset_status': valuelist_validator.overheid_dataset_status,
'overheid_taxonomiebeleidsagenda': valuelist_validator.overheid_taxonomiebeleidsagenda,
'overheid_openbaarheidsniveau': valuelist_validator.overheid_openbaarheidsniveau,
......
......@@ -81,7 +81,7 @@ def overheid_spatial_scheme(value):
:param value: str, the value to validate
:return: str, the original value if it passes validation
"""
return _check_values('overheid_spatial_scheme', value, 1, 0)
return _check_values('overheid_spatial_scheme', value, 1, 0, False)
def overheid_spatial_gemeente(value):
......@@ -252,7 +252,7 @@ def _check_value(name, value):
return value
def _check_values(name, values, minimum=1, maximum=0):
def _check_values(name, values, minimum=1, maximum=0, unique_check=True):
"""
Checks a given list of values against the given valuelist. Raises an
ckan.plugins.toolkit.Invalid if the values do not pass the validation.
......@@ -272,8 +272,9 @@ def _check_values(name, values, minimum=1, maximum=0):
if 0 < maximum < len(values):
raise tk.Invalid('values do not meet the maximum requirements')
if not len(values) == len(set(values)):
raise tk.Invalid('values must be unique')
if unique_check:
if not len(values) == len(set(values)):
raise tk.Invalid('values must be unique')
valid_values = _read_file_as_json(name)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment