Skip to content

Commit

Permalink
Support for ISO 19115 Part 3 XML (#933)
Browse files Browse the repository at this point in the history 
* Added initial code to ingest ISO 19115-3 XML

* Add ISO 19115-3 XML plugin

* Remove old files

* Rename profile

* Add support for output of vertical extent and funder

* Fix bugs in output code

* Fix incorrect namespace issue

* Get address info from contacts when GetCapabilities not supplied

* Add funder parsing, fix vert extent path

* Add initial tests

* Fix numerous path bugs

* Fix errors in constraints and spatial resolution

* Allowed anchor tag for identifiers

* Updated processing of image description and bands

* Add functional tests

* Add funder column to db

* Fix error im gml namespace path

* Add mdb unit test

* Cleaning up and adding comments

* Add mdb function comments

* Add db columns for vert extent

* Add function comments to profile code

* Fix vert extent and gml paths in tests

* Fix services bugs and add tests

* Update functional tests for new mdb XML test record

* Update comments and readme for ISO 19115 Part 3 XML

* Add mdb transaction tests

* Fixes:

- Missing contact name in converted records
- Add more tests

* Update due to functionality moved to OWSLib

* Updated tests and modifications for changed config format

* Update expected test responses after OWSLib funder changes

* Restore correct version of cite.db

* Remove cite.db from pull request

* Remove unnecessary trailing comma in setup.py

* Renamed func. test suites from 'mdb' to 'iso19115p3'

* Remove 'Funder' field and tests

* Correct the profile name in the iso19115p3 test suite

* Update iso19115p3 func. tests for profile name change
  • Loading branch information
@vjf
vjf committed Sep 20, 2024
1 parent 91353f4 commit b556ed5
Show file tree
Hide file tree
Showing 157 changed files with 26,197 additions and 25 deletions.
5 changes: 4 additions & 1 deletion pycsw/core/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ def __init__(self, prefix='csw30'):
'gmd': 'http://www.isotc211.org/2005/gmd',
'gml': 'http://www.opengis.net/gml',
'gml32': 'http://www.opengis.net/gml/3.2',
'mdb': 'http://standards.iso.org/iso/19115/-3/mdb/2.0',
'ogc': 'http://www.opengis.net/ogc',
'os': 'http://a9.com/-/spec/opensearch/1.1/',
'ows': 'http://www.opengis.net/ows',
Expand Down Expand Up @@ -118,7 +119,7 @@ def __init__(self, prefix='csw30'):
'pycsw:Metadata': 'metadata',
# raw metadata payload type, xml as default for now
'pycsw:MetadataType': 'metadata_type',
# bag of metadata element and attributes ONLY, no XML tages
# bag of metadata element and attributes ONLY, no XML tags
'pycsw:AnyText': 'anytext',
'pycsw:Language': 'language',
'pycsw:Title': 'title',
Expand All @@ -134,6 +135,8 @@ def __init__(self, prefix='csw30'):
'pycsw:Type': 'type',
# geometry, specified in OGC WKT
'pycsw:BoundingBox': 'wkt_geometry',
'pycsw:VertExtentMin': 'vert_extent_min',
'pycsw:VertExtentMax': 'vert_extent_max',
'pycsw:CRS': 'crs',
'pycsw:AlternateTitle': 'title_alternate',
'pycsw:RevisionDate': 'date_revision',
Expand Down
52 changes: 37 additions & 15 deletions pycsw/core/metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,9 @@ def _parse_metadata(context, repos, record):
return [_parse_dc(context, repos, exml)]
elif root == '{%s}DIF' % context.namespaces['dif']: # DIF
pass # TODO
elif root == '{%s}MD_Metadata' % context.namespaces['mdb']:
# ISO 19115p3 XML
return [_parse_iso(context, repos, exml)]
else:
raise RuntimeError('Unsupported metadata format')

Expand Down Expand Up @@ -1364,23 +1367,29 @@ def get_value_by_language(pt_group, language, pt_type='text'):
return recobj

def _parse_iso(context, repos, exml):
""" Parses ISO 19139, ISO 19115p3 """

from owslib.iso import MD_ImageDescription, MD_Metadata, SV_ServiceIdentification
from owslib.iso_che import CHE_MD_Metadata

recobj = repos.dataset()
bbox = None
links = []
mdmeta_ns = 'gmd'

if exml.tag == '{http://www.geocat.ch/2008/che}CHE_MD_Metadata':
md = CHE_MD_Metadata(exml)
elif exml.tag == '{http://standards.iso.org/iso/19115/-3/mdb/2.0}MD_Metadata':
from owslib.iso3 import MD_Metadata
md = MD_Metadata(exml)
mdmeta_ns = 'mdb'
else:
md = MD_Metadata(exml)

md_identification = md.identification[0]

_set(context, recobj, 'pycsw:Identifier', md.identifier)
_set(context, recobj, 'pycsw:Typename', 'gmd:MD_Metadata')
_set(context, recobj, 'pycsw:Typename', f'{mdmeta_ns}:MD_Metadata')
_set(context, recobj, 'pycsw:Schema', context.namespaces['gmd'])
_set(context, recobj, 'pycsw:MdSource', 'local')
_set(context, recobj, 'pycsw:InsertDate', util.get_today_and_now())
Expand All @@ -1394,7 +1403,7 @@ def _parse_iso(context, repos, exml):
_set(context, recobj, 'pycsw:Modified', md.datestamp)
_set(context, recobj, 'pycsw:Source', md.dataseturi)

if md.referencesystem is not None:
if md.referencesystem is not None and md.referencesystem.code is not None:
try:
code_ = 'urn:ogc:def:crs:EPSG::%d' % int(md.referencesystem.code)
except ValueError:
Expand All @@ -1421,11 +1430,21 @@ def _parse_iso(context, repos, exml):
elif len(md_identification.resourcelanguagecode) > 0:
_set(context, recobj, 'pycsw:ResourceLanguage', md_identification.resourcelanguagecode[0])

# Geographic bounding box
if hasattr(md_identification, 'bbox'):
bbox = md_identification.bbox
else:
bbox = None

# Vertical extent of a bounding box
if hasattr(md_identification, 'extent'):
if hasattr(md_identification.extent, 'vertExtMin') and \
md_identification.extent.vertExtMin is not None:
_set(context, recobj, 'pycsw:VertExtentMin', md_identification.extent.vertExtMin)
if hasattr(md_identification.extent, 'vertExtMax') and \
md_identification.extent.vertExtMax is not None:
_set(context, recobj, 'pycsw:VertExtentMax', md_identification.extent.vertExtMax)

if (hasattr(md_identification, 'keywords') and
len(md_identification.keywords) > 0):
all_keywords = [item for sublist in md_identification.keywords for item in sublist.keywords if item is not None]
Expand All @@ -1435,14 +1454,17 @@ def _parse_iso(context, repos, exml):
json.dumps([t for t in md_identification.keywords if t.thesaurus is not None],
default=lambda o: o.__dict__))

# Creator
if (hasattr(md_identification, 'creator') and
len(md_identification.creator) > 0):
all_orgs = set([item.organization for item in md_identification.creator if hasattr(item, 'organization') and item.organization is not None])
_set(context, recobj, 'pycsw:Creator', ';'.join(all_orgs))
# Publisher
if (hasattr(md_identification, 'publisher') and
len(md_identification.publisher) > 0):
all_orgs = set([item.organization for item in md_identification.publisher if hasattr(item, 'organization') and item.organization is not None])
_set(context, recobj, 'pycsw:Publisher', ';'.join(all_orgs))
# Contributor
if (hasattr(md_identification, 'contributor') and
len(md_identification.contributor) > 0):
all_orgs = set([item.organization for item in md_identification.contributor if hasattr(item, 'organization') and item.organization is not None])
Expand Down Expand Up @@ -1527,18 +1549,18 @@ def _parse_iso(context, repos, exml):

_set(context, recobj, 'pycsw:Keywords', keywords)

bands = []
for band in ci.bands:
band_info = {
'id': band.id,
'units': band.units,
'min': band.min,
'max': band.max
}
bands.append(band_info)
bands = []
for band in ci.bands:
band_info = {
'id': band.id,
'units': band.units,
'min': band.min,
'max': band.max
}
bands.append(band_info)

if len(bands) > 0:
_set(context, recobj, 'pycsw:Bands', json.dumps(bands))
if len(bands) > 0:
_set(context, recobj, 'pycsw:Bands', json.dumps(bands))

if hasattr(md, 'acquisition') and md.acquisition is not None:
platform = md.acquisition.platforms[0]
Expand All @@ -1557,10 +1579,10 @@ def _parse_iso(context, repos, exml):
if hasattr(md, 'distribution'):
dist_links = []
if hasattr(md.distribution, 'online'):
LOGGER.debug('Scanning for gmd:transferOptions element(s)')
LOGGER.debug(f'Scanning for {mdmeta_ns}:transferOptions element(s)')
dist_links.extend(md.distribution.online)
if hasattr(md.distribution, 'distributor'):
LOGGER.debug('Scanning for gmd:distributorTransferOptions element(s)')
LOGGER.debug(f'Scanning for {mdmeta_ns}:distributorTransferOptions element(s)')
for dist_member in md.distribution.distributor:
dist_links.extend(dist_member.online)
for link in dist_links:
Expand Down
7 changes: 5 additions & 2 deletions pycsw/core/repository.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,6 @@ def __init__(self, database, context, app_root=None, table='records', repo_filte
LOGGER.info('setting repository queryables')
# generate core queryables db and obj bindings
self.queryables = {}

for tname in self.context.model['typenames']:
for qname in self.context.model['typenames'][tname]['queryables']:
self.queryables[qname] = {}
Expand All @@ -231,7 +230,8 @@ def __init__(self, database, context, app_root=None, table='records', repo_filte
# TODO smarter way of doing this
self.queryables['_all'] = {}
for qbl in self.queryables:
self.queryables['_all'].update(self.queryables[qbl])
if qbl != '_all':
self.queryables['_all'].update(self.queryables[qbl])

self.queryables['_all'].update(self.context.md_core_model['mappings'])

Expand Down Expand Up @@ -708,6 +708,7 @@ def setup(database, table, create_sfsql_tables=True, postgis_geometry_column='wk
"""Setup database tables and indexes"""
from sqlalchemy import Column, create_engine, Integer, MetaData, \
Table, Text, Unicode
from sqlalchemy.types import Float
from sqlalchemy.orm import create_session

LOGGER.info('Creating database %s', database)
Expand Down Expand Up @@ -840,6 +841,8 @@ def setup(database, table, create_sfsql_tables=True, postgis_geometry_column='wk
Column('distancevalue', Text, index=True),
Column('distanceuom', Text, index=True),
Column('wkt_geometry', Text),
Column('vert_extent_min', Float, index=True),
Column('vert_extent_max', Float, index=True),

# service
Column('servicetype', Text, index=True),
Expand Down
29 changes: 29 additions & 0 deletions pycsw/plugins/profiles/iso19115p3/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
# -*- coding: utf-8 -*-
# =================================================================
#
# Authors: Tom Kralidis <[email protected]>
#
# Copyright (c) 2015 Tom Kralidis
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
# OTHER DEALINGS IN THE SOFTWARE.
#
# =================================================================
Loading

0 comments on commit b556ed5

Please sign in to comment.