GEOMESA-3392 Add avro-schema-registry converter

geomesa · Sep 13, 2024 · b48d480 · b48d480
1 parent 2875e29
commit b48d480
Show file tree

Hide file tree

Showing 10 changed files with 119 additions and 38 deletions.
diff --git a/build/list-nar.sh b/build/list-nar.sh
@@ -4,13 +4,12 @@
 dir="$(cd "$(dirname "$0")/.." || exit 1; pwd)"
 
 # find available nars
-mapfile -t nars < <( find "$dir"/geomesa-* -name "geomesa-*-nar" -type d | sort | sed 's|.*/geomesa-\([a-z0-9-]\+\)-nar|\1|' )
+mapfile -t nars < <( mvn -f "$dir" -q -am exec:exec -Dexec.executable="pwd" -T8 | grep -e '-nar$' | sort )
 
 function checkNar() {
   local nar="$1"
-  local desc="$2"
   if [[ -z "$nar" ]]; then
-    echo "No $desc nar found... try building with maven"
+    echo "No ${nar##*/} nar found... try building with maven"
     exit 1
   elif [[ $(echo "$nar" | wc -l) -gt 1 ]]; then
     echo -e "Found multiple nars: \n$nar"
@@ -35,9 +34,9 @@ if [[ -n "$1" ]]; then
 fi
 
 for nar in "${nars[@]}"; do
-  file="$(find "$dir"/geomesa-* -name "geomesa-${nar}-nar_*.nar")"
-  checkNar "$file" "$nar"
-  echo "$nar::"
+  file="$(find "$nar"/target -name '*.nar')"
+  checkNar "$file"
+  echo "${nar##*/}::"
   unzip -l "$file" | grep bundled-dependencies | sed 's|.*bundled-dependencies/||' | grep -v '^$' | sort
   echo ""
 done
diff --git a/geomesa-accumulo-bundle/geomesa-accumulo20-nar/pom.xml b/geomesa-accumulo-bundle/geomesa-accumulo20-nar/pom.xml
@@ -109,6 +109,10 @@
                     <groupId>io.netty</groupId>
                     <artifactId>netty-common</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-configuration2</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

diff --git a/geomesa-accumulo-bundle/geomesa-accumulo21-nar/pom.xml b/geomesa-accumulo-bundle/geomesa-accumulo21-nar/pom.xml
@@ -99,6 +99,14 @@
                     <groupId>io.netty</groupId>
                     <artifactId>netty-common</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.yaml</groupId>
+                    <artifactId>snakeyaml</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-configuration2</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

diff --git a/geomesa-datastore-bundle/geomesa-datastore-processors/pom.xml b/geomesa-datastore-bundle/geomesa-datastore-processors/pom.xml
@@ -55,6 +55,10 @@
             <groupId>org.locationtech.geomesa</groupId>
             <artifactId>geomesa-convert-all_${scala.binary.version}</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.locationtech.geomesa</groupId>
+            <artifactId>geomesa-convert-avro-schema-registry_${scala.binary.version}</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.locationtech.geomesa</groupId>
             <artifactId>geomesa-metrics-graphite_${scala.binary.version}</artifactId>

diff --git a/geomesa-datastore-bundle/geomesa-datastore-services-nar/pom.xml b/geomesa-datastore-bundle/geomesa-datastore-services-nar/pom.xml
@@ -102,6 +102,12 @@
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-text</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -132,10 +138,34 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>io.confluent</groupId>
+            <artifactId>kafka-schema-registry-client</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>commons-io</groupId>
+                    <artifactId>commons-io</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.commons</groupId>
+                    <artifactId>commons-lang3</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>io.confluent</groupId>
+            <artifactId>kafka-avro-serializer</artifactId>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.re2j</groupId>
+                    <artifactId>re2j</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>com.google.protobuf</groupId>
             <artifactId>protobuf-java</artifactId>
-            <version>${protobuf-java.version}</version>
+            <version>${protobuf.java.version}</version>
         </dependency>
 
         <!-- provided dependencies inherited from the geomesa-datastore-services-api-nar -->

diff --git a/geomesa-hbase-bundle/geomesa-hbase2-nar/pom.xml b/geomesa-hbase-bundle/geomesa-hbase2-nar/pom.xml
@@ -221,7 +221,7 @@
         <dependency>
             <groupId>com.google.protobuf</groupId>
             <artifactId>protobuf-java</artifactId>
-            <version>${protobuf-java.hbase.version}</version>
+            <version>${protobuf.java.hbase.version}</version>
         </dependency>
 
         <dependency>

diff --git a/geomesa-hbase-bundle/pom.xml b/geomesa-hbase-bundle/pom.xml
@@ -22,7 +22,7 @@
             <dependency>
                 <groupId>com.google.protobuf</groupId>
                 <artifactId>protobuf-java</artifactId>
-                <version>${protobuf-java.hbase.version}</version>
+                <version>${protobuf.java.hbase.version}</version>
             </dependency>
         </dependencies>
     </dependencyManagement>

diff --git a/geomesa-kafka-bundle/geomesa-kafka-nar/pom.xml b/geomesa-kafka-bundle/geomesa-kafka-nar/pom.xml
@@ -26,7 +26,7 @@
             <groupId>org.geomesa.nifi</groupId>
             <artifactId>geomesa-kafka-processors_${scala.binary.version}</artifactId>
             <exclusions>
-                <!-- exclude dependencies provided by datastore-services-api-nar -->
+                <!-- exclude dependencies provided by datastore-services-nar -->
                 <exclusion>
                     <groupId>org.scala-lang</groupId>
                     <artifactId>*</artifactId>
@@ -79,10 +79,18 @@
                     <groupId>commons-cli</groupId>
                     <artifactId>commons-cli</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>commons-io</groupId>
+                    <artifactId>commons-io</artifactId>
+                </exclusion>
                 <exclusion>
                     <groupId>com.github.luben</groupId>
                     <artifactId>zstd-jni</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.kafka</groupId>
+                    <artifactId>kafka-clients</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

diff --git a/geomesa-lambda-bundle/geomesa-lambda-nar/pom.xml b/geomesa-lambda-bundle/geomesa-lambda-nar/pom.xml
@@ -26,7 +26,7 @@
             <groupId>org.geomesa.nifi</groupId>
             <artifactId>geomesa-lambda-processors_${scala.binary.version}</artifactId>
             <exclusions>
-                <!-- exclude dependencies provided by datastore-services-api-nar -->
+                <!-- exclude dependencies provided by datastore-services-nar -->
                 <exclusion>
                     <groupId>org.scala-lang</groupId>
                     <artifactId>*</artifactId>
@@ -79,10 +79,18 @@
                     <groupId>commons-cli</groupId>
                     <artifactId>commons-cli</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>commons-io</groupId>
+                    <artifactId>commons-io</artifactId>
+                </exclusion>
                 <exclusion>
                     <groupId>com.github.luben</groupId>
                     <artifactId>zstd-jni</artifactId>
                 </exclusion>
+                <exclusion>
+                    <groupId>org.apache.kafka</groupId>
+                    <artifactId>kafka-clients</artifactId>
+                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

diff --git a/pom.xml b/pom.xml
@@ -82,33 +82,33 @@
         <zookeeper.version>3.9.1</zookeeper.version>
         <hadoop.version>3.3.6</hadoop.version>
         <hbase.version>2.5.7-hadoop3</hbase.version>
-        <kafka.version>3.7.0</kafka.version>
+        <kafka.version>3.7.1</kafka.version>
+        <confluent.version>7.7.1</confluent.version> <!-- corresponds to kafka 3.7 -->
         <postgres.version>42.7.2</postgres.version>
 
         <aircompressor.version>0.27</aircompressor.version>
+        <aws.java.sdk.version>1.12.625</aws.java.sdk.version>
+        <commons.beanutils.version>1.9.4</commons.beanutils.version>
+        <commons.configuration.version>1.10</commons.configuration.version>
+        <commons.configuration2.version>2.9.0</commons.configuration2.version>
+        <commons.io.version>2.16.1</commons.io.version>
+        <commons.lang3.version>3.14.0</commons.lang3.version>
+        <commons.net.version>3.10.0</commons.net.version>
+        <commons.text.version>1.11.0</commons.text.version>
         <curator.version>5.6.0</curator.version>
-        <netty.version>4.1.106.Final</netty.version>
-        <jetty.version>9.4.53.v20231009</jetty.version>
         <fasterxml.jackson.version>2.16.1</fasterxml.jackson.version>
-        <fasterxml.woodstox.version>5.4.0</fasterxml.woodstox.version> <!-- managed from 5.3.0 -->
-        <commons.net.version>3.10.0</commons.net.version> <!-- managed from 3.1 -->
-        <commons.text.version>1.11.0</commons.text.version>
-        <commons.lang3.version>3.14.0</commons.lang3.version>
-        <wildfly.openssl.version>1.1.3.Final</wildfly.openssl.version> <!-- managed from 1.0.7.Final -->
+        <fasterxml.woodstox.version>5.4.0</fasterxml.woodstox.version>
         <gson.version>2.10.1</gson.version>
-        <snappy-java.version>1.1.10.5</snappy-java.version> <!-- managed from 1.1.1.6 -->
-        <protobuf-java.version>3.25.1</protobuf-java.version>
-        <protobuf-java.hbase.version>2.6.1</protobuf-java.hbase.version> <!-- needs to be 2.x for hbase1 -->
-        <json-path.version>2.8.0</json-path.version> <!-- managed from 2.7.0 -->
-        <nimbus-jose-jwt.version>9.37.2</nimbus-jose-jwt.version> <!-- managed from 9.8.1 -->
-        <commons-configuration.version>1.10</commons-configuration.version> <!-- managed from 1.6 -->
-        <commons-configuration2.version>2.9.0</commons-configuration2.version> <!-- managed from 2.5.0 -->
-        <commons-beanutils.version>1.9.4</commons-beanutils.version> <!-- managed from 1.9.3 -->
         <httpclient.version>4.5.14</httpclient.version>
+        <jetty.version>9.4.53.v20231009</jetty.version>
+        <json.path.version>2.8.0</json.path.version>
+        <netty.version>4.1.106.Final</netty.version>
+        <nimbus.jose.jwt.version>9.37.2</nimbus.jose.jwt.version>
         <parquet.version>1.13.1</parquet.version>
-
-        <!-- aws version -->
-        <aws.java.sdk.version>1.12.625</aws.java.sdk.version>
+        <protobuf.java.version>3.25.1</protobuf.java.version>
+        <protobuf.java.hbase.version>2.6.1</protobuf.java.hbase.version> <!-- needs to be 2.x for hbase1 -->
+        <snappy.java.version>1.1.10.5</snappy.java.version>
+        <wildfly.openssl.version>1.1.3.Final</wildfly.openssl.version>
 
         <!-- logging -->
         <slf4j.version>1.7.36</slf4j.version>
@@ -126,7 +126,7 @@
         <test.redis.docker.tag>7-alpine</test.redis.docker.tag>
         <test.postgres.docker.tag>15.1</test.postgres.docker.tag>
         <test.postgis.docker.tag>15-3.4</test.postgis.docker.tag>
-        <test.confluent.docker.tag>7.6.0</test.confluent.docker.tag> <!-- confluent 7.6.x corresponds to kafka 3.6.x -->
+        <test.confluent.docker.tag>7.7.0</test.confluent.docker.tag> <!-- confluent 7.7.x corresponds to kafka 3.7.x -->
 
         <maven.test.opens>--add-opens=java.base/java.nio=ALL-UNNAMED --illegal-access=warn</maven.test.opens>
         <maven.test.jvmargs>-Xms512m -Xmx4g -XX:-UseGCOverheadLimit ${maven.test.opens}</maven.test.jvmargs>
@@ -380,6 +380,11 @@
                 <artifactId>geomesa-convert-all_${scala.binary.version}</artifactId>
                 <version>${geomesa.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.locationtech.geomesa</groupId>
+                <artifactId>geomesa-convert-avro-schema-registry_${scala.binary.version}</artifactId>
+                <version>${geomesa.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.locationtech.geomesa</groupId>
                 <artifactId>geomesa-index-api_${scala.binary.version}</artifactId>
@@ -605,6 +610,16 @@
                 <artifactId>kafka-clients</artifactId>
                 <version>${kafka.version}</version>
             </dependency>
+            <dependency>
+                <groupId>io.confluent</groupId>
+                <artifactId>kafka-schema-registry-client</artifactId>
+                <version>${confluent.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>io.confluent</groupId>
+                <artifactId>kafka-avro-serializer</artifactId>
+                <version>${confluent.version}</version>
+            </dependency>
 
             <dependency>
                 <groupId>org.geotools.jdbc</groupId>
@@ -768,7 +783,7 @@
             <dependency>
                 <groupId>com.jayway.jsonpath</groupId>
                 <artifactId>json-path</artifactId>
-                <version>${json-path.version}</version>
+                <version>${json.path.version}</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.httpcomponents</groupId>
@@ -780,7 +795,7 @@
             <dependency>
                 <groupId>org.xerial.snappy</groupId>
                 <artifactId>snappy-java</artifactId>
-                <version>${snappy-java.version}</version>
+                <version>${snappy.java.version}</version>
             </dependency>
             <dependency>
                 <groupId>com.fasterxml.jackson.core</groupId>
@@ -845,7 +860,7 @@
             <dependency>
                 <groupId>com.nimbusds</groupId>
                 <artifactId>nimbus-jose-jwt</artifactId>
-                <version>${nimbus-jose-jwt.version}</version>
+                <version>${nimbus.jose.jwt.version}</version>
             </dependency>
             <dependency>
                 <groupId>org.apache.curator</groupId>
@@ -890,18 +905,23 @@
             <dependency>
                 <groupId>org.apache.commons</groupId>
                 <artifactId>commons-configuration2</artifactId>
-                <version>${commons-configuration2.version}</version>
+                <version>${commons.configuration2.version}</version>
             </dependency>
             <dependency>
                 <groupId>com.google.protobuf</groupId>
                 <artifactId>protobuf-java</artifactId>
-                <version>${protobuf-java.version}</version>
+                <version>${protobuf.java.version}</version>
             </dependency>
             <dependency>
                 <groupId>io.airlift</groupId>
                 <artifactId>aircompressor</artifactId>
                 <version>${aircompressor.version}</version>
             </dependency>
+            <dependency>
+                <groupId>commons-io</groupId>
+                <artifactId>commons-io</artifactId>
+                <version>${commons.io.version}</version>
+            </dependency>
 
             <!-- Netty 4.x used by arrow, hadoop, hbase, and c* -->
             <dependency>
@@ -990,12 +1010,12 @@
             <dependency>
                 <groupId>commons-beanutils</groupId>
                 <artifactId>commons-beanutils</artifactId>
-                <version>${commons-beanutils.version}</version>
+                <version>${commons.beanutils.version}</version>
             </dependency>
             <dependency>
                 <groupId>commons-configuration</groupId>
                 <artifactId>commons-configuration</artifactId>
-                <version>${commons-configuration.version}</version>
+                <version>${commons.configuration.version}</version>
                 <exclusions>
                     <exclusion>
                         <!-- exclude commons-beanutils-core, which doesn't exist in 1.9.3 -->