From 2add4e20812394447650249ad0cfa98792832f16 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Thu, 9 Mar 2023 22:03:23 +0000 Subject: [PATCH 01/18] feat(i18n): start locale support; - Started locale support with usage of the get_translated helper. --- ckanext/dcat/converters.py | 30 ++++++++++++++++---- ckanext/dcat/profiles.py | 58 ++++++++++++++++++++++++++++++-------- 2 files changed, 70 insertions(+), 18 deletions(-) diff --git a/ckanext/dcat/converters.py b/ckanext/dcat/converters.py index ddce1723..5eb16e48 100644 --- a/ckanext/dcat/converters.py +++ b/ckanext/dcat/converters.py @@ -1,4 +1,13 @@ from past.builtins import basestring +from ckan.plugins.toolkit import ( + h, + get_action, + g, + abort, + _ +) +from ckan import model +from ckan.logic import NotFound import logging log = logging.getLogger(__name__) @@ -58,13 +67,15 @@ def ckan_to_dcat(package_dict): dcat_dict = {} - dcat_dict['title'] = package_dict.get('title') - dcat_dict['description'] = package_dict.get('notes') - dcat_dict['landingPage'] = package_dict.get('url') + dcat_dict['title'] = h.get_translated(package_dict, 'title') + dcat_dict['description'] = h.get_translated(package_dict, 'notes') + dcat_dict['landingPage'] = h.url_for(package_dict.get('type', 'dataset') \ + + '.read', id=package_dict.get('id')) dcat_dict['keyword'] = [] - for tag in package_dict.get('tags', []): + tags = h.get_translated(package_dict, 'tags') + for tag in tags: dcat_dict['keyword'].append(tag['name']) @@ -93,9 +104,16 @@ def ckan_to_dcat(package_dict): dcat_dict['distribution'] = [] for resource in package_dict.get('resources', []): + context = {u'model': model, u'session': model.Session, + u'user': g.user, u'auth_user_obj': g.userobj} + try: + res_dict = get_action(u'resource_show')(context, {'id': resource.get('id')}) + except NotFound: + abort(404, _('Resource %s does not exist.' % resource.get('id'))) + distribution = { - 'title': resource.get('name'), - 'description': resource.get('description'), + 'title': h.get_translated(res_dict, 'name'), + 'description': h.get_translated(res_dict, 'description'), 'format': resource.get('format'), 'byteSize': resource.get('size'), # TODO: downloadURL or accessURL depending on resource type? diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py index 9f18408d..bdd6802b 100644 --- a/ckanext/dcat/profiles.py +++ b/ckanext/dcat/profiles.py @@ -19,6 +19,8 @@ from ckan.model.license import LicenseRegister from ckan.plugins import toolkit +from ckan import model +from ckan.logic import NotFound from ckan.lib.munge import munge_tag from ckanext.dcat.urls import url_for from ckanext.dcat.utils import resource_uri, publisher_uri_organization_fallback, DCAT_EXPOSE_SUBCATALOGS, DCAT_CLEAN_TAGS @@ -157,7 +159,7 @@ def _keywords(self, dataset_ref): ''' Returns all DCAT keywords on a particular dataset ''' - keywords = self._object_value_list(dataset_ref, DCAT.keyword) or [] + keywords = toolkit.h.get_translated(dataset_ref, 'tags') # Split keywords with commas keywords_with_commas = [k for k in keywords if ',' in k] for keyword in keywords_with_commas: @@ -991,15 +993,21 @@ def parse_dataset(self, dataset_dict, dataset_ref): # Basic fields for key, predicate in ( - ('title', DCT.title), - ('notes', DCT.description), - ('url', DCAT.landingPage), ('version', OWL.versionInfo), ): value = self._object_value(dataset_ref, predicate) if value: dataset_dict[key] = value + # Fluent URL workaround + url = url_for(dataset_dict.get('type', 'dataset') \ + + '.read', id=dataset_dict.get('id'), _external=True) + dataset_dict['url'] = url + + # translation workaround + dataset_dict['title'] = toolkit.h.get_translated(dataset_dict, 'title') + dataset_dict['notes'] = toolkit.h.get_translated(dataset_dict, 'notes') + if not dataset_dict.get('version'): # adms:version was supported on the first version of the DCAT-AP value = self._object_value(dataset_ref, ADMS.version) @@ -1209,9 +1217,6 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): # Basic fields items = [ - ('title', DCT.title, None, Literal), - ('notes', DCT.description, None, Literal), - ('url', DCAT.landingPage, None, URIRef), ('identifier', DCT.identifier, ['guid', 'id'], URIRefOrLiteral), ('version', OWL.versionInfo, ['dcat_version'], Literal), ('version_notes', ADMS.versionNotes, None, Literal), @@ -1222,8 +1227,20 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): ] self._add_triples_from_dict(dataset_dict, dataset_ref, items) + # Fluent URL workaround + url = url_for(dataset_dict.get('type', 'dataset') \ + + '.read', id=dataset_dict.get('id'), _external=True) + g.add((dataset_ref, DCAT.landingPage, URIRef(url))) + + # translation workaround + title = toolkit.h.get_translated(dataset_dict, 'title') + notes = toolkit.h.get_translated(dataset_dict, 'notes') + g.add((dataset_ref, DCT.title, Literal(title))) + g.add((dataset_ref, DCT.description, Literal(notes))) + # Tags - for tag in dataset_dict.get('tags', []): + tags = toolkit.h.get_translated(dataset_dict, 'tags') + for tag in tags: g.add((dataset_ref, DCAT.keyword, Literal(tag['name']))) # Dates @@ -1286,10 +1303,16 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): self._get_dataset_value(dataset_dict, 'publisher_name'), dataset_dict.get('organization'), ]): + context = {u'model': model, u'session': model.Session, + u'user': toolkit.g.user, u'auth_user_obj': toolkit.g.userobj} + try: + org_dict = toolkit.get_action(u'organization_show')(context, {u'id': dataset_dict['organization']['id']}) + except NotFound: + toolkit.abort(404, toolkit._('Organization not found')) - publisher_uri = self._get_dataset_value(dataset_dict, 'publisher_uri') + publisher_uri = toolkit.h.url_for('organization.read', id=org_dict.get('id')) publisher_uri_fallback = publisher_uri_organization_fallback(dataset_dict) - publisher_name = self._get_dataset_value(dataset_dict, 'publisher_name') + publisher_name = toolkit.h.get_translated(org_dict, 'title') if publisher_uri: publisher_details = CleanedURIRef(publisher_uri) elif not publisher_name and publisher_uri_fallback: @@ -1366,8 +1389,6 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): # Simple values items = [ - ('name', DCT.title, None, Literal), - ('description', DCT.description, None, Literal), ('status', ADMS.status, None, URIRefOrLiteral), ('rights', DCT.rights, None, URIRefOrLiteral), ('license', DCT.license, None, URIRefOrLiteral), @@ -1377,6 +1398,19 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): self._add_triples_from_dict(resource_dict, distribution, items) + # translation workaround + context = {u'model': model, u'session': model.Session, + u'user': toolkit.g.user, u'auth_user_obj': toolkit.g.userobj} + try: + res_dict = toolkit.get_action(u'resource_show')(context, {'id': resource_dict.get('id')}) + except NotFound: + toolkit.abort(404, toolkit._('Resource %s does not exist.' % resource_dict.get('id'))) + + name = toolkit.h.get_translated(res_dict, 'name') + description = toolkit.h.get_translated(res_dict, 'description') or '' + g.add((distribution, DCT.title, Literal(name))) + g.add((distribution, DCT.description, Literal(description))) + # Lists items = [ ('documentation', FOAF.page, None, URIRefOrLiteral), From 86273827152c7fe42ea16b597156281aad84d492 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Fri, 10 Mar 2023 21:51:51 +0000 Subject: [PATCH 02/18] fix(i18n): tag fix; - Fixed keywords. --- ckanext/dcat/profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py index bdd6802b..05b54e9a 100644 --- a/ckanext/dcat/profiles.py +++ b/ckanext/dcat/profiles.py @@ -1241,7 +1241,7 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): # Tags tags = toolkit.h.get_translated(dataset_dict, 'tags') for tag in tags: - g.add((dataset_ref, DCAT.keyword, Literal(tag['name']))) + g.add((dataset_ref, DCAT.keyword, Literal(tag))) # Dates items = [ From b1f5bfcf47ee37dd1224c41e8e1956cef6751870 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Thu, 13 Apr 2023 20:44:50 +0000 Subject: [PATCH 03/18] fix(resources): removed unnecessary code; - Removed unnecessary imports and code for the resource translations. --- ckanext/dcat/converters.py | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/ckanext/dcat/converters.py b/ckanext/dcat/converters.py index 5eb16e48..a377a222 100644 --- a/ckanext/dcat/converters.py +++ b/ckanext/dcat/converters.py @@ -1,13 +1,5 @@ from past.builtins import basestring -from ckan.plugins.toolkit import ( - h, - get_action, - g, - abort, - _ -) -from ckan import model -from ckan.logic import NotFound +from ckan.plugins.toolkit import h import logging log = logging.getLogger(__name__) @@ -104,16 +96,9 @@ def ckan_to_dcat(package_dict): dcat_dict['distribution'] = [] for resource in package_dict.get('resources', []): - context = {u'model': model, u'session': model.Session, - u'user': g.user, u'auth_user_obj': g.userobj} - try: - res_dict = get_action(u'resource_show')(context, {'id': resource.get('id')}) - except NotFound: - abort(404, _('Resource %s does not exist.' % resource.get('id'))) - distribution = { - 'title': h.get_translated(res_dict, 'name'), - 'description': h.get_translated(res_dict, 'description'), + 'title': h.get_translated(resource, 'name'), + 'description': h.get_translated(resource, 'description'), 'format': resource.get('format'), 'byteSize': resource.get('size'), # TODO: downloadURL or accessURL depending on resource type? From 9e9c6ec2292203b0a475a6c72aecd728ddfefe87 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Mon, 17 Apr 2023 20:00:20 +0000 Subject: [PATCH 04/18] revert(i18n): reverted profile changes; - Reverted profile.py back to master. --- ckanext/dcat/profiles.py | 60 +++++++++------------------------------- 1 file changed, 13 insertions(+), 47 deletions(-) diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py index 05b54e9a..9f18408d 100644 --- a/ckanext/dcat/profiles.py +++ b/ckanext/dcat/profiles.py @@ -19,8 +19,6 @@ from ckan.model.license import LicenseRegister from ckan.plugins import toolkit -from ckan import model -from ckan.logic import NotFound from ckan.lib.munge import munge_tag from ckanext.dcat.urls import url_for from ckanext.dcat.utils import resource_uri, publisher_uri_organization_fallback, DCAT_EXPOSE_SUBCATALOGS, DCAT_CLEAN_TAGS @@ -159,7 +157,7 @@ def _keywords(self, dataset_ref): ''' Returns all DCAT keywords on a particular dataset ''' - keywords = toolkit.h.get_translated(dataset_ref, 'tags') + keywords = self._object_value_list(dataset_ref, DCAT.keyword) or [] # Split keywords with commas keywords_with_commas = [k for k in keywords if ',' in k] for keyword in keywords_with_commas: @@ -993,21 +991,15 @@ def parse_dataset(self, dataset_dict, dataset_ref): # Basic fields for key, predicate in ( + ('title', DCT.title), + ('notes', DCT.description), + ('url', DCAT.landingPage), ('version', OWL.versionInfo), ): value = self._object_value(dataset_ref, predicate) if value: dataset_dict[key] = value - # Fluent URL workaround - url = url_for(dataset_dict.get('type', 'dataset') \ - + '.read', id=dataset_dict.get('id'), _external=True) - dataset_dict['url'] = url - - # translation workaround - dataset_dict['title'] = toolkit.h.get_translated(dataset_dict, 'title') - dataset_dict['notes'] = toolkit.h.get_translated(dataset_dict, 'notes') - if not dataset_dict.get('version'): # adms:version was supported on the first version of the DCAT-AP value = self._object_value(dataset_ref, ADMS.version) @@ -1217,6 +1209,9 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): # Basic fields items = [ + ('title', DCT.title, None, Literal), + ('notes', DCT.description, None, Literal), + ('url', DCAT.landingPage, None, URIRef), ('identifier', DCT.identifier, ['guid', 'id'], URIRefOrLiteral), ('version', OWL.versionInfo, ['dcat_version'], Literal), ('version_notes', ADMS.versionNotes, None, Literal), @@ -1227,21 +1222,9 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): ] self._add_triples_from_dict(dataset_dict, dataset_ref, items) - # Fluent URL workaround - url = url_for(dataset_dict.get('type', 'dataset') \ - + '.read', id=dataset_dict.get('id'), _external=True) - g.add((dataset_ref, DCAT.landingPage, URIRef(url))) - - # translation workaround - title = toolkit.h.get_translated(dataset_dict, 'title') - notes = toolkit.h.get_translated(dataset_dict, 'notes') - g.add((dataset_ref, DCT.title, Literal(title))) - g.add((dataset_ref, DCT.description, Literal(notes))) - # Tags - tags = toolkit.h.get_translated(dataset_dict, 'tags') - for tag in tags: - g.add((dataset_ref, DCAT.keyword, Literal(tag))) + for tag in dataset_dict.get('tags', []): + g.add((dataset_ref, DCAT.keyword, Literal(tag['name']))) # Dates items = [ @@ -1303,16 +1286,10 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): self._get_dataset_value(dataset_dict, 'publisher_name'), dataset_dict.get('organization'), ]): - context = {u'model': model, u'session': model.Session, - u'user': toolkit.g.user, u'auth_user_obj': toolkit.g.userobj} - try: - org_dict = toolkit.get_action(u'organization_show')(context, {u'id': dataset_dict['organization']['id']}) - except NotFound: - toolkit.abort(404, toolkit._('Organization not found')) - publisher_uri = toolkit.h.url_for('organization.read', id=org_dict.get('id')) + publisher_uri = self._get_dataset_value(dataset_dict, 'publisher_uri') publisher_uri_fallback = publisher_uri_organization_fallback(dataset_dict) - publisher_name = toolkit.h.get_translated(org_dict, 'title') + publisher_name = self._get_dataset_value(dataset_dict, 'publisher_name') if publisher_uri: publisher_details = CleanedURIRef(publisher_uri) elif not publisher_name and publisher_uri_fallback: @@ -1389,6 +1366,8 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): # Simple values items = [ + ('name', DCT.title, None, Literal), + ('description', DCT.description, None, Literal), ('status', ADMS.status, None, URIRefOrLiteral), ('rights', DCT.rights, None, URIRefOrLiteral), ('license', DCT.license, None, URIRefOrLiteral), @@ -1398,19 +1377,6 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): self._add_triples_from_dict(resource_dict, distribution, items) - # translation workaround - context = {u'model': model, u'session': model.Session, - u'user': toolkit.g.user, u'auth_user_obj': toolkit.g.userobj} - try: - res_dict = toolkit.get_action(u'resource_show')(context, {'id': resource_dict.get('id')}) - except NotFound: - toolkit.abort(404, toolkit._('Resource %s does not exist.' % resource_dict.get('id'))) - - name = toolkit.h.get_translated(res_dict, 'name') - description = toolkit.h.get_translated(res_dict, 'description') or '' - g.add((distribution, DCT.title, Literal(name))) - g.add((distribution, DCT.description, Literal(description))) - # Lists items = [ ('documentation', FOAF.page, None, URIRefOrLiteral), From 5722d40a81075b08f2096e1cf2b71796a89c84df Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Mon, 17 Apr 2023 20:47:37 +0000 Subject: [PATCH 05/18] feat(i18n): modified dcat_dataset_show method; - Modified code to condition on fluent being loaded. - Added code to the `dcat_dataset_show` method to set the values of the data_dict before the serializer profiles. --- ckanext/dcat/converters.py | 28 ++++++++++++++++++++-------- ckanext/dcat/logic.py | 26 +++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/ckanext/dcat/converters.py b/ckanext/dcat/converters.py index a377a222..e0fb4251 100644 --- a/ckanext/dcat/converters.py +++ b/ckanext/dcat/converters.py @@ -1,5 +1,6 @@ from past.builtins import basestring from ckan.plugins.toolkit import h +from ckan.plugins import plugin_loaded import logging log = logging.getLogger(__name__) @@ -59,18 +60,25 @@ def ckan_to_dcat(package_dict): dcat_dict = {} - dcat_dict['title'] = h.get_translated(package_dict, 'title') - dcat_dict['description'] = h.get_translated(package_dict, 'notes') - dcat_dict['landingPage'] = h.url_for(package_dict.get('type', 'dataset') \ - + '.read', id=package_dict.get('id')) + dcat_dict['title'] = package_dict.get('title') + dcat_dict['description'] = package_dict.get('notes') + dcat_dict['landingPage'] = package_dict.get('url') dcat_dict['keyword'] = [] - tags = h.get_translated(package_dict, 'tags') - for tag in tags: + for tag in package_dict.get('tags', []): dcat_dict['keyword'].append(tag['name']) + # Fluent compatibility + if plugin_loaded('fluent'): + dcat_dict['title'] = h.get_translated(package_dict, 'title') + dcat_dict['description'] = h.get_translated(package_dict, 'notes') + tags = h.get_translated(package_dict, 'tags') + for tag in tags: + dcat_dict['keyword'].append(tag['name']) + + dcat_dict['publisher'] = {} for extra in package_dict.get('extras', []): @@ -97,13 +105,17 @@ def ckan_to_dcat(package_dict): dcat_dict['distribution'] = [] for resource in package_dict.get('resources', []): distribution = { - 'title': h.get_translated(resource, 'name'), - 'description': h.get_translated(resource, 'description'), + 'title': resource.get('name'), + 'description': resource.get('description'), 'format': resource.get('format'), 'byteSize': resource.get('size'), # TODO: downloadURL or accessURL depending on resource type? 'accessURL': resource.get('url'), } + # Fluent compatibility + if plugin_loaded('fluent'): + distribution['title'] = h.get_translated(resource, 'name') + distribution['description'] = h.get_translated(resource, 'description') dcat_dict['distribution'].append(distribution) return dcat_dict diff --git a/ckanext/dcat/logic.py b/ckanext/dcat/logic.py index 33f6e652..6c82855b 100644 --- a/ckanext/dcat/logic.py +++ b/ckanext/dcat/logic.py @@ -5,7 +5,7 @@ from ckantoolkit import config from dateutil.parser import parse as dateutil_parse -from ckan.plugins import toolkit +from ckan.plugins import toolkit, plugin_loaded import ckanext.dcat.converters as converters @@ -24,6 +24,30 @@ def dcat_dataset_show(context, data_dict): dataset_dict = toolkit.get_action('package_show')(context, data_dict) + # Fluent compatibility + if plugin_loaded('fluent'): + # basic dataset fields + dataset_dict['title'] = toolkit.h.get_translated(dataset_dict, 'title') + dataset_dict['notes'] = toolkit.h.get_translated(dataset_dict, 'notes') + dataset_dict['tags'] = toolkit.h.get_translated(dataset_dict, 'tags') + + # dataset resources + for resource in dataset_dict.get('resources', []): + resource['name'] = toolkit.h.get_translated(resource, 'name') + resource['description'] = toolkit.h.get_translated(resource, 'description') + + # dataset organization + if 'organization' in dataset_dict \ + and 'id' in dataset_dict['organization']: + try: + org_dict = toolkit.get_action(u'organization_show')( + {u'user': toolkit.g.user}, + {u'id': dataset_dict['organization']['id']}) + except toolkit.ObjectNotFound: + toolkit.abort(404, toolkit._('Organization not found')) + dataset_dict['organization']['title'] = toolkit.h.get_translated(org_dict, 'title') + dataset_dict['organization']['notes'] = toolkit.h.get_translated(org_dict, 'notes') + serializer = RDFSerializer(profiles=data_dict.get('profiles')) output = serializer.serialize_dataset(dataset_dict, From 93169d4e6810594d8da06e08d512ebce9c103569 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Tue, 18 Apr 2023 16:45:54 +0000 Subject: [PATCH 06/18] feat(i18n): added fluent support to catalog; - Added code for fluent fields to catalog show method. --- ckanext/dcat/logic.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/ckanext/dcat/logic.py b/ckanext/dcat/logic.py index 6c82855b..8966736b 100644 --- a/ckanext/dcat/logic.py +++ b/ckanext/dcat/logic.py @@ -65,6 +65,31 @@ def dcat_catalog_show(context, data_dict): dataset_dicts = query['results'] pagination_info = _pagination_info(query, data_dict) + # Fluent compatibility + if plugin_loaded('fluent'): + for dataset_dict in dataset_dicts: + # basic dataset fields + dataset_dict['title'] = toolkit.h.get_translated(dataset_dict, 'title') + dataset_dict['notes'] = toolkit.h.get_translated(dataset_dict, 'notes') + dataset_dict['tags'] = toolkit.h.get_translated(dataset_dict, 'tags') + + # dataset resources + for resource in dataset_dict.get('resources', []): + resource['name'] = toolkit.h.get_translated(resource, 'name') + resource['description'] = toolkit.h.get_translated(resource, 'description') + + # dataset organization + if 'organization' in dataset_dict \ + and 'id' in dataset_dict['organization']: + try: + org_dict = toolkit.get_action(u'organization_show')( + {u'user': toolkit.g.user}, + {u'id': dataset_dict['organization']['id']}) + except toolkit.ObjectNotFound: + toolkit.abort(404, toolkit._('Organization not found')) + dataset_dict['organization']['title'] = toolkit.h.get_translated(org_dict, 'title') + dataset_dict['organization']['notes'] = toolkit.h.get_translated(org_dict, 'notes') + serializer = RDFSerializer(profiles=data_dict.get('profiles')) output = serializer.serialize_catalog({}, dataset_dicts, From ad1bfde3a713fcf224a77d72ec56a6576eb08fd9 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Tue, 18 Apr 2023 19:01:23 +0000 Subject: [PATCH 07/18] feat(i18n): added language replacement in dcat.base_uri config value; - Added replacement of `{{LANG}}` to the `ckanext.dcat.base_uri` config value. --- ckanext/dcat/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ckanext/dcat/utils.py b/ckanext/dcat/utils.py index b86b64c9..927c9ca8 100644 --- a/ckanext/dcat/utils.py +++ b/ckanext/dcat/utils.py @@ -20,6 +20,8 @@ class HelperError(Exception): from ckan import model import ckan.plugins.toolkit as toolkit +from ckan.plugins import plugin_loaded +from flask import has_request_context from ckanext.dcat.exceptions import RDFProfileException @@ -158,6 +160,8 @@ def catalog_uri(): ''' uri = config.get('ckanext.dcat.base_uri') + if uri and plugin_loaded('fluent') and has_request_context(): + uri = re.sub('{{LANG}}', str(toolkit.h.lang()), uri) if not uri: uri = config.get('ckan.site_url') if not uri: From 907249e369930c73df96cc7fed1e1787bd4f3611 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Tue, 18 Apr 2023 19:05:33 +0000 Subject: [PATCH 08/18] fix(i18n): moved fluent support for catalog uri; - Moved `{{LANG}}` replacement down. --- ckanext/dcat/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ckanext/dcat/utils.py b/ckanext/dcat/utils.py index 927c9ca8..7edad04b 100644 --- a/ckanext/dcat/utils.py +++ b/ckanext/dcat/utils.py @@ -160,8 +160,6 @@ def catalog_uri(): ''' uri = config.get('ckanext.dcat.base_uri') - if uri and plugin_loaded('fluent') and has_request_context(): - uri = re.sub('{{LANG}}', str(toolkit.h.lang()), uri) if not uri: uri = config.get('ckan.site_url') if not uri: @@ -176,6 +174,9 @@ def catalog_uri(): 'the `ckanext.dcat.base_uri` or `ckan.site_url` ' + 'option') + if uri and plugin_loaded('fluent') and has_request_context(): + uri = re.sub('{{LANG}}', str(toolkit.h.lang()), uri) + return uri From 821062e5c03616da26f5fcce9cfa090c8999d2bf Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Tue, 9 May 2023 13:51:38 +0000 Subject: [PATCH 09/18] revert(i18n): reverted fluent comp. in converters and logic; - Removed fluent compatibility code in converters. - Removed fluent compatibility code in logic. - Removed fluent loaded check in `catalog_uri` util method. --- ckanext/dcat/converters.py | 15 ----------- ckanext/dcat/logic.py | 51 +------------------------------------- ckanext/dcat/utils.py | 3 +-- 3 files changed, 2 insertions(+), 67 deletions(-) diff --git a/ckanext/dcat/converters.py b/ckanext/dcat/converters.py index e0fb4251..ddce1723 100644 --- a/ckanext/dcat/converters.py +++ b/ckanext/dcat/converters.py @@ -1,6 +1,4 @@ from past.builtins import basestring -from ckan.plugins.toolkit import h -from ckan.plugins import plugin_loaded import logging log = logging.getLogger(__name__) @@ -70,15 +68,6 @@ def ckan_to_dcat(package_dict): dcat_dict['keyword'].append(tag['name']) - # Fluent compatibility - if plugin_loaded('fluent'): - dcat_dict['title'] = h.get_translated(package_dict, 'title') - dcat_dict['description'] = h.get_translated(package_dict, 'notes') - tags = h.get_translated(package_dict, 'tags') - for tag in tags: - dcat_dict['keyword'].append(tag['name']) - - dcat_dict['publisher'] = {} for extra in package_dict.get('extras', []): @@ -112,10 +101,6 @@ def ckan_to_dcat(package_dict): # TODO: downloadURL or accessURL depending on resource type? 'accessURL': resource.get('url'), } - # Fluent compatibility - if plugin_loaded('fluent'): - distribution['title'] = h.get_translated(resource, 'name') - distribution['description'] = h.get_translated(resource, 'description') dcat_dict['distribution'].append(distribution) return dcat_dict diff --git a/ckanext/dcat/logic.py b/ckanext/dcat/logic.py index 8966736b..33f6e652 100644 --- a/ckanext/dcat/logic.py +++ b/ckanext/dcat/logic.py @@ -5,7 +5,7 @@ from ckantoolkit import config from dateutil.parser import parse as dateutil_parse -from ckan.plugins import toolkit, plugin_loaded +from ckan.plugins import toolkit import ckanext.dcat.converters as converters @@ -24,30 +24,6 @@ def dcat_dataset_show(context, data_dict): dataset_dict = toolkit.get_action('package_show')(context, data_dict) - # Fluent compatibility - if plugin_loaded('fluent'): - # basic dataset fields - dataset_dict['title'] = toolkit.h.get_translated(dataset_dict, 'title') - dataset_dict['notes'] = toolkit.h.get_translated(dataset_dict, 'notes') - dataset_dict['tags'] = toolkit.h.get_translated(dataset_dict, 'tags') - - # dataset resources - for resource in dataset_dict.get('resources', []): - resource['name'] = toolkit.h.get_translated(resource, 'name') - resource['description'] = toolkit.h.get_translated(resource, 'description') - - # dataset organization - if 'organization' in dataset_dict \ - and 'id' in dataset_dict['organization']: - try: - org_dict = toolkit.get_action(u'organization_show')( - {u'user': toolkit.g.user}, - {u'id': dataset_dict['organization']['id']}) - except toolkit.ObjectNotFound: - toolkit.abort(404, toolkit._('Organization not found')) - dataset_dict['organization']['title'] = toolkit.h.get_translated(org_dict, 'title') - dataset_dict['organization']['notes'] = toolkit.h.get_translated(org_dict, 'notes') - serializer = RDFSerializer(profiles=data_dict.get('profiles')) output = serializer.serialize_dataset(dataset_dict, @@ -65,31 +41,6 @@ def dcat_catalog_show(context, data_dict): dataset_dicts = query['results'] pagination_info = _pagination_info(query, data_dict) - # Fluent compatibility - if plugin_loaded('fluent'): - for dataset_dict in dataset_dicts: - # basic dataset fields - dataset_dict['title'] = toolkit.h.get_translated(dataset_dict, 'title') - dataset_dict['notes'] = toolkit.h.get_translated(dataset_dict, 'notes') - dataset_dict['tags'] = toolkit.h.get_translated(dataset_dict, 'tags') - - # dataset resources - for resource in dataset_dict.get('resources', []): - resource['name'] = toolkit.h.get_translated(resource, 'name') - resource['description'] = toolkit.h.get_translated(resource, 'description') - - # dataset organization - if 'organization' in dataset_dict \ - and 'id' in dataset_dict['organization']: - try: - org_dict = toolkit.get_action(u'organization_show')( - {u'user': toolkit.g.user}, - {u'id': dataset_dict['organization']['id']}) - except toolkit.ObjectNotFound: - toolkit.abort(404, toolkit._('Organization not found')) - dataset_dict['organization']['title'] = toolkit.h.get_translated(org_dict, 'title') - dataset_dict['organization']['notes'] = toolkit.h.get_translated(org_dict, 'notes') - serializer = RDFSerializer(profiles=data_dict.get('profiles')) output = serializer.serialize_catalog({}, dataset_dicts, diff --git a/ckanext/dcat/utils.py b/ckanext/dcat/utils.py index 7edad04b..f3e426ee 100644 --- a/ckanext/dcat/utils.py +++ b/ckanext/dcat/utils.py @@ -20,7 +20,6 @@ class HelperError(Exception): from ckan import model import ckan.plugins.toolkit as toolkit -from ckan.plugins import plugin_loaded from flask import has_request_context from ckanext.dcat.exceptions import RDFProfileException @@ -174,7 +173,7 @@ def catalog_uri(): 'the `ckanext.dcat.base_uri` or `ckan.site_url` ' + 'option') - if uri and plugin_loaded('fluent') and has_request_context(): + if uri and has_request_context(): uri = re.sub('{{LANG}}', str(toolkit.h.lang()), uri) return uri From 4e82e8122b976c72adbc44083128d6e4fe0b5a26 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Tue, 9 May 2023 16:47:43 +0000 Subject: [PATCH 10/18] feat(i18n): added fluent support to profiles; - Removed `LANG` replacement in base uri util method. - Added multilingual support to graph from dataset. --- ckanext/dcat/profiles.py | 56 ++++++++++++++++++++++++++++++---------- ckanext/dcat/utils.py | 4 --- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py index 9f18408d..b6569d47 100644 --- a/ckanext/dcat/profiles.py +++ b/ckanext/dcat/profiles.py @@ -18,7 +18,7 @@ from geomet import wkt, InvalidGeoJSONException from ckan.model.license import LicenseRegister -from ckan.plugins import toolkit +from ckan.plugins import toolkit, plugin_loaded from ckan.lib.munge import munge_tag from ckanext.dcat.urls import url_for from ckanext.dcat.utils import resource_uri, publisher_uri_organization_fallback, DCAT_EXPOSE_SUBCATALOGS, DCAT_CLEAN_TAGS @@ -727,14 +727,16 @@ def _add_list_triples_from_dict(self, _dict, subject, items): def _add_triples_from_dict(self, _dict, subject, items, list_value=False, - date_value=False): + date_value=False, + multilingual=False): for item in items: key, predicate, fallbacks, _type = item self._add_triple_from_dict(_dict, subject, predicate, key, fallbacks=fallbacks, list_value=list_value, date_value=date_value, - _type=_type) + _type=_type, + multilingual=multilingual) def _add_triple_from_dict(self, _dict, subject, predicate, key, fallbacks=None, @@ -742,7 +744,8 @@ def _add_triple_from_dict(self, _dict, subject, predicate, key, date_value=False, _type=Literal, _datatype=None, - value_modifier=None): + value_modifier=None, + multilingual=False): ''' Adds a new triple to the graph with the provided parameters @@ -776,6 +779,12 @@ def _add_triple_from_dict(self, _dict, subject, predicate, key, self._add_date_triple(subject, predicate, value, _type) elif value: # Normal text value + if multilingual and isinstance(value, dict): + # We assume that all multilingual field values are Literals + for lang, translated_value in value.items(): + object = Literal(translated_value, lang=lang) + self.g.add((subject, predicate, object)) + return # ensure URIRef items are preprocessed (space removal/url encoding) if _type == URIRef: _type = CleanedURIRef @@ -1207,10 +1216,16 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): g.add((dataset_ref, RDF.type, DCAT.Dataset)) + # Multilingual fields + multilingual_suffix = '_translated' if plugin_loaded('fluent') else '' + items = [ + ('title%s' % multilingual_suffix, DCT.title, None, Literal), + ('notes%s' % multilingual_suffix, DCT.description, None, Literal), + ] + self._add_triples_from_dict(dataset_dict, dataset_ref, items, multilingual=True) + # Basic fields items = [ - ('title', DCT.title, None, Literal), - ('notes', DCT.description, None, Literal), ('url', DCAT.landingPage, None, URIRef), ('identifier', DCT.identifier, ['guid', 'id'], URIRefOrLiteral), ('version', OWL.versionInfo, ['dcat_version'], Literal), @@ -1223,8 +1238,12 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): self._add_triples_from_dict(dataset_dict, dataset_ref, items) # Tags - for tag in dataset_dict.get('tags', []): - g.add((dataset_ref, DCAT.keyword, Literal(tag['name']))) + for tag in dataset_dict.get('tags%s' % multilingual_suffix, dataset_dict.get('tags', [])): + for lang, translated_value in tag.items(): + if lang not in config.get('ckan.locales_offered'): + g.add((dataset_ref, DCAT.keyword, Literal(tag['name']))) + break + g.add((dataset_ref, DCAT.keyword, Literal(translated_value['name'], lang=lang))) # Dates items = [ @@ -1306,9 +1325,15 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): # If no name but an URI is available, the name literal remains empty to # avoid mixing organization and dataset values. if not publisher_name and not publisher_uri and dataset_dict.get('organization'): - publisher_name = dataset_dict['organization']['title'] - - g.add((publisher_details, FOAF.name, Literal(publisher_name))) + try: + org_dict = toolkit.get_action(u'organization_show')({u'user': toolkit.g.user}, + {u'id': dataset_dict['organization']['id']}) + items = [('title%s' % multilingual_suffix, FOAF.name, None, Literal)] + self._add_triples_from_dict(org_dict, publisher_details, items, multilingual=True) + except toolkit.ObjectNotFound: + pass + else: + g.add((publisher_details, FOAF.name, Literal(publisher_name))) # TODO: It would make sense to fallback these to organization # fields but they are not in the default schema and the # `organization` object in the dataset_dict does not include @@ -1364,10 +1389,15 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): g.add((distribution, RDF.type, DCAT.Distribution)) + # Multilingual fields + items = [ + ('name%s' % multilingual_suffix, DCT.title, None, Literal), + ('description%s' % multilingual_suffix, DCT.description, None, Literal), + ] + self._add_triples_from_dict(resource_dict, distribution, items, multilingual=True) + # Simple values items = [ - ('name', DCT.title, None, Literal), - ('description', DCT.description, None, Literal), ('status', ADMS.status, None, URIRefOrLiteral), ('rights', DCT.rights, None, URIRefOrLiteral), ('license', DCT.license, None, URIRefOrLiteral), diff --git a/ckanext/dcat/utils.py b/ckanext/dcat/utils.py index f3e426ee..b86b64c9 100644 --- a/ckanext/dcat/utils.py +++ b/ckanext/dcat/utils.py @@ -20,7 +20,6 @@ class HelperError(Exception): from ckan import model import ckan.plugins.toolkit as toolkit -from flask import has_request_context from ckanext.dcat.exceptions import RDFProfileException @@ -173,9 +172,6 @@ def catalog_uri(): 'the `ckanext.dcat.base_uri` or `ckan.site_url` ' + 'option') - if uri and has_request_context(): - uri = re.sub('{{LANG}}', str(toolkit.h.lang()), uri) - return uri From de51a74ff266d3230c2e3993e82707674b05f6a7 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Tue, 9 May 2023 17:08:30 +0000 Subject: [PATCH 11/18] feat(i18n): made translated fields act more like the get translated helper; - Made the translation keys fallback to the normal core field, like the `get_translated` helper does. --- ckanext/dcat/profiles.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py index b6569d47..c4ed4b69 100644 --- a/ckanext/dcat/profiles.py +++ b/ckanext/dcat/profiles.py @@ -18,7 +18,7 @@ from geomet import wkt, InvalidGeoJSONException from ckan.model.license import LicenseRegister -from ckan.plugins import toolkit, plugin_loaded +from ckan.plugins import toolkit from ckan.lib.munge import munge_tag from ckanext.dcat.urls import url_for from ckanext.dcat.utils import resource_uri, publisher_uri_organization_fallback, DCAT_EXPOSE_SUBCATALOGS, DCAT_CLEAN_TAGS @@ -1217,10 +1217,11 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): g.add((dataset_ref, RDF.type, DCAT.Dataset)) # Multilingual fields - multilingual_suffix = '_translated' if plugin_loaded('fluent') else '' + title_key = 'title_translated' if 'title_translated' in dataset_dict else 'title' + notes_key = 'notes_translated' if 'notes_translated' in dataset_dict else 'notes' items = [ - ('title%s' % multilingual_suffix, DCT.title, None, Literal), - ('notes%s' % multilingual_suffix, DCT.description, None, Literal), + (title_key, DCT.title, None, Literal), + (notes_key, DCT.description, None, Literal), ] self._add_triples_from_dict(dataset_dict, dataset_ref, items, multilingual=True) @@ -1238,7 +1239,7 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): self._add_triples_from_dict(dataset_dict, dataset_ref, items) # Tags - for tag in dataset_dict.get('tags%s' % multilingual_suffix, dataset_dict.get('tags', [])): + for tag in dataset_dict.get('tags_translated', dataset_dict.get('tags', [])): for lang, translated_value in tag.items(): if lang not in config.get('ckan.locales_offered'): g.add((dataset_ref, DCAT.keyword, Literal(tag['name']))) @@ -1328,7 +1329,8 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): try: org_dict = toolkit.get_action(u'organization_show')({u'user': toolkit.g.user}, {u'id': dataset_dict['organization']['id']}) - items = [('title%s' % multilingual_suffix, FOAF.name, None, Literal)] + title_key = 'title_translated' if 'title_translated' in org_dict else 'title' + items = [(title_key, FOAF.name, None, Literal)] self._add_triples_from_dict(org_dict, publisher_details, items, multilingual=True) except toolkit.ObjectNotFound: pass @@ -1390,9 +1392,11 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): g.add((distribution, RDF.type, DCAT.Distribution)) # Multilingual fields + name_key = 'name_translated' if 'name_translated' in resource_dict else 'name' + description_key = 'description_translated' if 'description_translated' in resource_dict else 'description' items = [ - ('name%s' % multilingual_suffix, DCT.title, None, Literal), - ('description%s' % multilingual_suffix, DCT.description, None, Literal), + (name_key, DCT.title, None, Literal), + (description_key, DCT.description, None, Literal), ] self._add_triples_from_dict(resource_dict, distribution, items, multilingual=True) From 1582a06da5f774c11ae6b55f9ef4737fd3c36e8f Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Tue, 9 May 2023 17:18:23 +0000 Subject: [PATCH 12/18] fix(i18n): better tags condition; - Changed conditions for translated tags. --- ckanext/dcat/profiles.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py index c4ed4b69..ef090c3f 100644 --- a/ckanext/dcat/profiles.py +++ b/ckanext/dcat/profiles.py @@ -1240,11 +1240,11 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): # Tags for tag in dataset_dict.get('tags_translated', dataset_dict.get('tags', [])): - for lang, translated_value in tag.items(): - if lang not in config.get('ckan.locales_offered'): - g.add((dataset_ref, DCAT.keyword, Literal(tag['name']))) - break - g.add((dataset_ref, DCAT.keyword, Literal(translated_value['name'], lang=lang))) + if 'name' in tag: + g.add((dataset_ref, DCAT.keyword, Literal(tag['name']))) + else: + for lang, translated_value in tag.items(): + g.add((dataset_ref, DCAT.keyword, Literal(translated_value['name'], lang=lang))) # Dates items = [ From 0551784f1ef50765d409e1321237a105a7f4ac83 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Wed, 10 May 2023 15:34:26 +0000 Subject: [PATCH 13/18] refactor(i18n): renamed multilingual to all_translated; - Renamed `multilingual` to `all_translated` due to core extension name. --- ckanext/dcat/profiles.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py index ef090c3f..2b8db515 100644 --- a/ckanext/dcat/profiles.py +++ b/ckanext/dcat/profiles.py @@ -728,7 +728,7 @@ def _add_list_triples_from_dict(self, _dict, subject, items): def _add_triples_from_dict(self, _dict, subject, items, list_value=False, date_value=False, - multilingual=False): + all_translated=False): for item in items: key, predicate, fallbacks, _type = item self._add_triple_from_dict(_dict, subject, predicate, key, @@ -736,7 +736,7 @@ def _add_triples_from_dict(self, _dict, subject, items, list_value=list_value, date_value=date_value, _type=_type, - multilingual=multilingual) + all_translated=multilingual) def _add_triple_from_dict(self, _dict, subject, predicate, key, fallbacks=None, @@ -745,7 +745,7 @@ def _add_triple_from_dict(self, _dict, subject, predicate, key, _type=Literal, _datatype=None, value_modifier=None, - multilingual=False): + all_translated=False): ''' Adds a new triple to the graph with the provided parameters @@ -1223,7 +1223,7 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): (title_key, DCT.title, None, Literal), (notes_key, DCT.description, None, Literal), ] - self._add_triples_from_dict(dataset_dict, dataset_ref, items, multilingual=True) + self._add_triples_from_dict(dataset_dict, dataset_ref, items, all_translated=True) # Basic fields items = [ @@ -1331,7 +1331,7 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): {u'id': dataset_dict['organization']['id']}) title_key = 'title_translated' if 'title_translated' in org_dict else 'title' items = [(title_key, FOAF.name, None, Literal)] - self._add_triples_from_dict(org_dict, publisher_details, items, multilingual=True) + self._add_triples_from_dict(org_dict, publisher_details, items, all_translated=True) except toolkit.ObjectNotFound: pass else: @@ -1398,7 +1398,7 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): (name_key, DCT.title, None, Literal), (description_key, DCT.description, None, Literal), ] - self._add_triples_from_dict(resource_dict, distribution, items, multilingual=True) + self._add_triples_from_dict(resource_dict, distribution, items, all_translated=True) # Simple values items = [ From 5971f9126b7ecef380cfac384ce4a118149975c4 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Wed, 10 May 2023 16:23:57 +0000 Subject: [PATCH 14/18] fix(i18n): syntax error; - Fixed syntax error from refactor. --- ckanext/dcat/profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py index 2b8db515..e03a1461 100644 --- a/ckanext/dcat/profiles.py +++ b/ckanext/dcat/profiles.py @@ -736,7 +736,7 @@ def _add_triples_from_dict(self, _dict, subject, items, list_value=list_value, date_value=date_value, _type=_type, - all_translated=multilingual) + all_translated=all_translated) def _add_triple_from_dict(self, _dict, subject, predicate, key, fallbacks=None, From 9e5f2cf3282cf94c9be9820e46d8c9070c0e98bf Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Wed, 10 May 2023 16:31:28 +0000 Subject: [PATCH 15/18] fix(i18n): syntax error; - Fixed syntax error from refactor. --- ckanext/dcat/profiles.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py index e03a1461..6947bb43 100644 --- a/ckanext/dcat/profiles.py +++ b/ckanext/dcat/profiles.py @@ -779,8 +779,8 @@ def _add_triple_from_dict(self, _dict, subject, predicate, key, self._add_date_triple(subject, predicate, value, _type) elif value: # Normal text value - if multilingual and isinstance(value, dict): - # We assume that all multilingual field values are Literals + if all_translated and isinstance(value, dict): + # We assume that all translated field values are Literals for lang, translated_value in value.items(): object = Literal(translated_value, lang=lang) self.g.add((subject, predicate, object)) From fb998444bbe8496dfea8f95930beb604e5a927c8 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Wed, 10 May 2023 17:02:35 +0000 Subject: [PATCH 16/18] fix(i18n): fluent tags output; - Fixed fluent tags output. --- ckanext/dcat/profiles.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py index 6947bb43..6c9d4922 100644 --- a/ckanext/dcat/profiles.py +++ b/ckanext/dcat/profiles.py @@ -1244,7 +1244,7 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): g.add((dataset_ref, DCAT.keyword, Literal(tag['name']))) else: for lang, translated_value in tag.items(): - g.add((dataset_ref, DCAT.keyword, Literal(translated_value['name'], lang=lang))) + g.add((dataset_ref, DCAT.keyword, Literal(translated_value, lang=lang))) # Dates items = [ From ab55002f438af088680cdbda4dd04c0e3226dcf5 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Wed, 10 May 2023 17:36:14 +0000 Subject: [PATCH 17/18] fix(i18n): fluent tags output; - Fixed fluent tags output. --- ckanext/dcat/profiles.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py index 6c9d4922..ba561ff3 100644 --- a/ckanext/dcat/profiles.py +++ b/ckanext/dcat/profiles.py @@ -1239,12 +1239,14 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): self._add_triples_from_dict(dataset_dict, dataset_ref, items) # Tags - for tag in dataset_dict.get('tags_translated', dataset_dict.get('tags', [])): + tags = dataset_dict.get('tags_translated', dataset_dict.get('tags', [])) + for tag in tags: if 'name' in tag: g.add((dataset_ref, DCAT.keyword, Literal(tag['name']))) else: - for lang, translated_value in tag.items(): - g.add((dataset_ref, DCAT.keyword, Literal(translated_value, lang=lang))) + # translated tags are stored as {'lang': ['tag1', 'tag2', ...]} + for translated_value in tags[tag]: + g.add((dataset_ref, DCAT.keyword, Literal(translated_value, lang=tag))) # Dates items = [ From 1d1b1553847eaa2cf9d72809c597dc330717ae26 Mon Sep 17 00:00:00 2001 From: Jesse Vickery Date: Wed, 8 Nov 2023 18:25:16 +0000 Subject: [PATCH 18/18] feat(dev): org dict cache, test fix, new var name; - Renamed `all_translated` to `translated`. - Added a class cache variable for org dicts. - Updated org test for serializing. --- ckanext/dcat/profiles.py | 34 ++++++++++++------- .../test_euro_dcatap_profile_serialize.py | 10 +++--- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/ckanext/dcat/profiles.py b/ckanext/dcat/profiles.py index ba561ff3..2e7fe17d 100644 --- a/ckanext/dcat/profiles.py +++ b/ckanext/dcat/profiles.py @@ -133,6 +133,9 @@ def __init__(self, graph, compatibility_mode=False): # _license(). self._licenceregister_cache = None + # Cache for Organizations + self._org_cache = None + def _datasets(self): ''' Generator that returns all DCAT datasets on the graph @@ -728,7 +731,7 @@ def _add_list_triples_from_dict(self, _dict, subject, items): def _add_triples_from_dict(self, _dict, subject, items, list_value=False, date_value=False, - all_translated=False): + translated=False): for item in items: key, predicate, fallbacks, _type = item self._add_triple_from_dict(_dict, subject, predicate, key, @@ -736,7 +739,7 @@ def _add_triples_from_dict(self, _dict, subject, items, list_value=list_value, date_value=date_value, _type=_type, - all_translated=all_translated) + translated=translated) def _add_triple_from_dict(self, _dict, subject, predicate, key, fallbacks=None, @@ -745,7 +748,7 @@ def _add_triple_from_dict(self, _dict, subject, predicate, key, _type=Literal, _datatype=None, value_modifier=None, - all_translated=False): + translated=False): ''' Adds a new triple to the graph with the provided parameters @@ -779,7 +782,7 @@ def _add_triple_from_dict(self, _dict, subject, predicate, key, self._add_date_triple(subject, predicate, value, _type) elif value: # Normal text value - if all_translated and isinstance(value, dict): + if translated and isinstance(value, dict): # We assume that all translated field values are Literals for lang, translated_value in value.items(): object = Literal(translated_value, lang=lang) @@ -1223,7 +1226,7 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): (title_key, DCT.title, None, Literal), (notes_key, DCT.description, None, Literal), ] - self._add_triples_from_dict(dataset_dict, dataset_ref, items, all_translated=True) + self._add_triples_from_dict(dataset_dict, dataset_ref, items, translated=True) # Basic fields items = [ @@ -1328,14 +1331,21 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): # If no name but an URI is available, the name literal remains empty to # avoid mixing organization and dataset values. if not publisher_name and not publisher_uri and dataset_dict.get('organization'): - try: - org_dict = toolkit.get_action(u'organization_show')({u'user': toolkit.g.user}, - {u'id': dataset_dict['organization']['id']}) + org_id = dataset_dict["organization"]["id"] + org_dict = None + if org_id in self._org_cache: + org_dict = self._org_cache[org_id] + else: + try: + org_dict = toolkit.get_action(u'organization_show')({u'ignore_auth': True}, + {u'id': org_id}) + self._org_cache[org_id] = org_dict + except toolkit.ObjectNotFound: + pass + if org_dict: title_key = 'title_translated' if 'title_translated' in org_dict else 'title' items = [(title_key, FOAF.name, None, Literal)] - self._add_triples_from_dict(org_dict, publisher_details, items, all_translated=True) - except toolkit.ObjectNotFound: - pass + self._add_triples_from_dict(org_dict, publisher_details, items, translated=True) else: g.add((publisher_details, FOAF.name, Literal(publisher_name))) # TODO: It would make sense to fallback these to organization @@ -1400,7 +1410,7 @@ def graph_from_dataset(self, dataset_dict, dataset_ref): (name_key, DCT.title, None, Literal), (description_key, DCT.description, None, Literal), ] - self._add_triples_from_dict(resource_dict, distribution, items, all_translated=True) + self._add_triples_from_dict(resource_dict, distribution, items, translated=True) # Simple values items = [ diff --git a/ckanext/dcat/tests/test_euro_dcatap_profile_serialize.py b/ckanext/dcat/tests/test_euro_dcatap_profile_serialize.py index e0f7d3c0..8944fc38 100644 --- a/ckanext/dcat/tests/test_euro_dcatap_profile_serialize.py +++ b/ckanext/dcat/tests/test_euro_dcatap_profile_serialize.py @@ -17,7 +17,7 @@ from ckanext.dcat import utils from ckanext.dcat.processors import RDFSerializer from ckanext.dcat.profiles import (DCAT, DCT, ADMS, XSD, VCARD, FOAF, SCHEMA, - SKOS, LOCN, GSP, OWL, SPDX, GEOJSON_IMT, + SKOS, LOCN, GSP, OWL, SPDX, GEOJSON_IMT, DISTRIBUTION_LICENSE_FALLBACK_CONFIG) from ckanext.dcat.utils import DCAT_EXPOSE_SUBCATALOGS from ckanext.dcat.tests.utils import BaseSerializeTest @@ -398,13 +398,15 @@ def test_publisher_extras(self): assert self._triple(g, publisher, DCT.type, URIRef(extras['publisher_type'])) def test_publisher_org(self): + org = factories.Organization() + dataset = { 'id': '4b6fe9ca-dc77-4cec-92a4-55c6624a5bd6', 'name': 'test-dataset', 'organization': { - 'id': '', - 'name': 'publisher1', - 'title': 'Example Publisher from Org', + 'id': org['id'], + 'name': org['name'], + 'title': org['title'], } }