sunileman / spark-kafka-streaming Goto Github PK

[warn] There may be incompatibilities among your library dependencies; run 'evicted' to see detailed eviction warnings.
[info] compiling 24 Scala sources to /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/target/scala-2.11/classes ...
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadKafkaGeoS3.scala:47:33: value geoapidata is not a member of object com.cloudera.examples.covidschema
[error]       val jsonStr = covidschema.geoapidata
[error]                                 ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadKafkaGeoS3.scala:68:17: ambiguous reference to overloaded definition,
[error] both method from_json in object functions of type (e: org.apache.spark.sql.Column, schema: org.apache.spark.sql.Column)org.apache.spark.sql.Column
[error] and  method from_json in object functions of type (e: org.apache.spark.sql.Column, schema: org.apache.spark.sql.types.StructType)org.apache.spark.sql.Column
[error] match argument types (org.apache.spark.sql.ColumnName,schema: <error>)
[error]         .select(from_json($"value", schema = broadcastSchema.value).as("data"))
[error]                 ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadKafkaGeoS3Local.scala:42:33: value geoapidata is not a member of object com.cloudera.examples.covidschema
[error]       val jsonStr = covidschema.geoapidata
[error]                                 ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadKafkaGeoS3Local.scala:62:17: ambiguous reference to overloaded definition,
[error] both method from_json in object functions of type (e: org.apache.spark.sql.Column, schema: org.apache.spark.sql.Column)org.apache.spark.sql.Column
[error] and  method from_json in object functions of type (e: org.apache.spark.sql.Column, schema: org.apache.spark.sql.types.StructType)org.apache.spark.sql.Column
[error] match argument types (org.apache.spark.sql.ColumnName,schema: <error>)
[error]         .select(from_json($"value", schema = broadcastSchema.value).as("data"))
[error]                 ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadReadGeoS3.scala:5:12: object amazonaws is not a member of package com
[error] import com.amazonaws.services.s3.AmazonS3ClientBuilder
[error]            ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadReadGeoS3.scala:6:12: object amazonaws is not a member of package com
[error] import com.amazonaws.auth.AWSStaticCredentialsProvider
[error]            ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadReadGeoS3.scala:7:12: object amazonaws is not a member of package com
[error] import com.amazonaws.auth.BasicAWSCredentials
[error]            ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadReadGeoS3.scala:10:12: object amazonaws is not a member of package com
[error] import com.amazonaws.services.s3.iterable.S3Objects
[error]            ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadReadGeoS3.scala:11:12: object amazonaws is not a member of package com
[error] import com.amazonaws.services.s3.model.CopyObjectRequest
[error]            ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadReadGeoS3.scala:12:12: object typesafe is not a member of package com
[error] import com.typesafe.config.ConfigFactory
[error]            ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadReadGeoS3.scala:36:20: not found: value AmazonS3ClientBuilder
[error]     val s3Client = AmazonS3ClientBuilder.standard().build();
[error]                    ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadReadGeoS3.scala:57:23: not found: value S3Objects
[error]       for (summary <- S3Objects.withPrefix(s3Client, sources3bucket, sourcefolder)) {
[error]                       ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoadReadGeoS3.scala:70:26: not found: type CopyObjectRequest
[error]           val creq = new CopyObjectRequest(sources3bucket, objkey, targets3bucket, targetfolder + "/" + filename)
[error]                          ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoveS3Files.scala:7:12: object amazonaws is not a member of package com
[error] import com.amazonaws.services.s3.AmazonS3ClientBuilder
[error]            ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoveS3Files.scala:8:12: object amazonaws is not a member of package com
[error] import com.amazonaws.services.s3.iterable.S3Objects
[error]            ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoveS3Files.scala:9:12: object amazonaws is not a member of package com
[error] import com.amazonaws.services.s3.model.CopyObjectRequest
[error]            ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoveS3Files.scala:48:20: not found: value AmazonS3ClientBuilder
[error]     val s3Client = AmazonS3ClientBuilder.standard().build();
[error]                    ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoveS3Files.scala:68:21: not found: value S3Objects
[error]     for (summary <- S3Objects.withPrefix(s3Client, sources3bucket, sourcefolder)) {
[error]                     ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/MoveS3Files.scala:76:24: not found: type CopyObjectRequest
[error]         val creq = new CopyObjectRequest(sources3bucket, objkey, targets3bucket, targetfolder + "/" + filename)
[error]                        ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/moadBingTransform.scala:25:30: value bingschema is not a member of object com.cloudera.examples.covidschema
[error]     val schema = covidschema.bingschema
[error]                              ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/moadTravelDataTransform.scala:25:30: value travelschema is not a member of object com.cloudera.examples.covidschema
[error]     val schema = covidschema.travelschema
[error]                              ^
[error] /Users/dchaffelson/IdeaProjects/spark-kafka-streaming/src/main/scala/com/cloudera/examples/readlocaljson.scala:21:31: value covidjhapi is not a member of object com.cloudera.examples.covidschema
[error]     val jsonStr = covidschema.covidjhapi
[error]                               ^
[error] 22 errors found
[error] (Compile / compileIncremental) Compilation failed
[error] Total time: 3 s, completed 26-Apr-2021 22:10:39
[IJ]