Skip to content

Commit a58ae7f

Browse files
Ting Yuannlu90syhily
authored
[Release] Migrate pulsar-spark from bintray to maven central (#56)
* Migrate pulsar-spark from bintray to maven central Co-authored-by: Neng Lu <[email protected]> Co-authored-by: Yufan Sheng <[email protected]>
1 parent c467f09 commit a58ae7f

17 files changed

+240
-743
lines changed

‎.github/workflows/release.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
name: Pulsar-Spark Release
2+
3+
on:
4+
release:
5+
types: [created]
6+
7+
jobs:
8+
upload:
9+
name: Upload Release files
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: actions/checkout@v2
13+
- name: Set up JDK 1.8
14+
uses: actions/setup-java@v1
15+
with:
16+
java-version: 1.8
17+
18+
- name: Release Maven package
19+
uses: samuelmeuli/action-maven-publish@v1
20+
with:
21+
gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }}
22+
gpg_passphrase: ${{ secrets.PASSPHRASE }}
23+
nexus_username: ${{ secrets.SONATYPE_USERNAME }}
24+
nexus_password: ${{ secrets.SONATYPE_PASSWORD }}
25+
server_id: ossrh
26+
maven_profiles: release
27+
maven_goals_phases: clean deploy
28+
maven_args: -DskipTests

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,3 +53,4 @@ logs/*
5353

5454
# shaded
5555
**/dependency-reduced-pom.xml
56+
**/pom.xml.versionsBackup

‎pom.xml‎

Lines changed: 71 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,12 @@
1919
<parent>
2020
<groupId>org.apache</groupId>
2121
<artifactId>apache</artifactId>
22-
<version>18</version>
22+
<version>24</version>
2323
</parent>
2424

2525
<groupId>io.streamnative.connectors</groupId>
26-
<artifactId>pulsar-spark-connector_${scala.binary.version}</artifactId>
27-
<version>2.4.6-SNAPSHOT</version>
26+
<artifactId>pulsar-spark-connector_2.12</artifactId>
27+
<version>3.1.1.1-SNAPSHOT</version>
2828
<name>StreamNative :: Pulsar Spark Connector</name>
2929
<url>https://pulsar.apache.org</url>
3030
<inceptionYear>2019</inceptionYear>
@@ -39,7 +39,7 @@
3939
<connection>scm:git:https://github.com/streamnative/pulsar-spark.git</connection>
4040
<developerConnection>scm:git:https://github.com/streamnative/pulsar-spark.git</developerConnection>
4141
<url>https://github.com/streamnative/pulsar-spark</url>
42-
<tag>branch-2.4.0</tag>
42+
<tag>master</tag>
4343
</scm>
4444
<issueManagement>
4545
<system>Github</system>
@@ -52,32 +52,36 @@
5252
</developer>
5353
</developers>
5454
<properties>
55+
<project.build.outputTimestamp>1</project.build.outputTimestamp>
5556
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
5657
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
5758
<project.scm.id>snbot-github</project.scm.id>
5859
<javac.target>1.8</javac.target>
60+
<maven.compiler.source>1.8</maven.compiler.source>
61+
<maven.compiler.target>1.8</maven.compiler.target>
5962
<redirectTestOutputToFile>true</redirectTestOutputToFile>
6063
<testRetryCount>2</testRetryCount>
6164

6265
<!-- dependencies -->
6366
<!-- latest version from apache pulsar master -->
6467
<pulsar.version>2.4.2</pulsar.version>
65-
<scala.version>2.11.12</scala.version>
66-
<scala.binary.version>2.11</scala.binary.version>
67-
<scalatest.version>3.0.3</scalatest.version>
68-
<spark.version>2.4.2</spark.version>
68+
<scala.version>2.12.10</scala.version>
69+
<scala.binary.version>2.12</scala.binary.version>
70+
<scalatest.version>3.2.2</scalatest.version>
71+
<spark.version>3.1.1</spark.version>
6972
<streamnative-tests.version>2.4.2</streamnative-tests.version>
7073
<testcontainers.version>1.10.6</testcontainers.version>
7174
<!-- plugin dependencies -->
7275
<maven.version>3.5.4</maven.version>
73-
<license-maven-plugin.version>3.0.rc1</license-maven-plugin.version>
74-
<maven-compiler-plugin.version>3.7.0</maven-compiler-plugin.version>
75-
<maven-dependency-plugin.version>3.1.1</maven-dependency-plugin.version>
76-
<maven-shade-plugin.version>3.1.0</maven-shade-plugin.version>
77-
<maven-source-plugin.version>2.2.1</maven-source-plugin.version>
78-
<maven-surefire-plugin.version>2.21.0</maven-surefire-plugin.version>
79-
<scala-maven-plugin.version>4.0.2</scala-maven-plugin.version>
80-
<scalatest-maven-plugin.version>2.0.0</scalatest-maven-plugin.version>
76+
<license-maven-plugin.version>3.0</license-maven-plugin.version>
77+
<maven-compiler-plugin.version>3.8.1</maven-compiler-plugin.version>
78+
<maven-dependency-plugin.version>3.2.0</maven-dependency-plugin.version>
79+
<maven-shade-plugin.version>3.2.4</maven-shade-plugin.version>
80+
<maven-source-plugin.version>3.2.0</maven-source-plugin.version>
81+
<maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version>
82+
<scala-maven-plugin.version>4.4.1</scala-maven-plugin.version>
83+
<scalatest-maven-plugin.version>2.0.2</scalatest-maven-plugin.version>
84+
<nexus-staging-maven-plugin.version>1.6.8</nexus-staging-maven-plugin.version>
8185
</properties>
8286

8387
<!-- dependencies for all modules -->
@@ -216,6 +220,12 @@
216220
<artifactId>scala-maven-plugin</artifactId>
217221
<version>${scala-maven-plugin.version}</version>
218222
<executions>
223+
<execution>
224+
<id>attach-javadocs</id>
225+
<goals>
226+
<goal>doc-jar</goal>
227+
</goals>
228+
</execution>
219229
<execution>
220230
<id>eclipse-add-source</id>
221231
<goals>
@@ -365,8 +375,6 @@
365375
</plugin>
366376
</plugins>
367377
</build>
368-
<profiles>
369-
</profiles>
370378
<repositories>
371379
<repository>
372380
<id>central</id>
@@ -380,9 +388,52 @@
380388
</repository>
381389
</repositories>
382390
<distributionManagement>
391+
<snapshotRepository>
392+
<id>ossrh</id>
393+
<url>https://oss.sonatype.org/content/repositories/snapshots</url>
394+
</snapshotRepository>
383395
<repository>
384-
<id>streamnative</id>
385-
<url>https://api.bintray.com/maven/streamnative/maven/io.streamnative.pulsar-spark/;publish=1</url>
396+
<id>ossrh</id>
397+
<url>https://oss.sonatype.org/service/local/staging/deploy/maven2/</url>
386398
</repository>
387399
</distributionManagement>
400+
<profiles>
401+
<profile>
402+
<id>release</id>
403+
<build>
404+
<plugins>
405+
<plugin>
406+
<artifactId>maven-gpg-plugin</artifactId>
407+
<executions>
408+
<execution>
409+
<id>sign-artifacts</id>
410+
<phase>verify</phase>
411+
<goals>
412+
<goal>sign</goal>
413+
</goals>
414+
</execution>
415+
</executions>
416+
<configuration>
417+
<!-- Prevent `gpg` from using pinentry programs -->
418+
<gpgArguments>
419+
<arg>--pinentry-mode</arg>
420+
<arg>loopback</arg>
421+
</gpgArguments>
422+
</configuration>
423+
</plugin>
424+
<plugin>
425+
<groupId>org.sonatype.plugins</groupId>
426+
<artifactId>nexus-staging-maven-plugin</artifactId>
427+
<version>${nexus-staging-maven-plugin.version}</version>
428+
<extensions>true</extensions>
429+
<configuration>
430+
<serverId>ossrh</serverId>
431+
<nexusUrl>https://oss.sonatype.org/</nexusUrl>
432+
<autoReleaseAfterClose>true</autoReleaseAfterClose>
433+
</configuration>
434+
</plugin>
435+
</plugins>
436+
</build>
437+
</profile>
438+
</profiles>
388439
</project>

‎src/main/scala/org/apache/spark/sql/pulsar/JSONParser.scala‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -53,8 +53,7 @@ class JacksonRecordParser(schema: DataType, val options: JSONOptions) extends Lo
5353
private val rootConverter =
5454
makeStructRootConverter(schema.asInstanceOf[StructType])
5555

56-
private val factory = new JsonFactory()
57-
options.setJacksonOptions(factory)
56+
private val factory = options.buildJsonFactory()
5857

5958
private def makeStructRootConverter(
6059
st: StructType): (JsonParser, InternalRow) => InternalRow = {
@@ -161,11 +160,13 @@ class JacksonRecordParser(schema: DataType, val options: JSONOptions) extends Lo
161160
// This one will lose microseconds parts.
162161
// See https://issues.apache.org/jira/browse/SPARK-10681.
163162
Long.box {
164-
Try(options.timestampFormat.parse(stringValue).getTime * 1000L)
163+
Try(TimestampFormatter(options.timestampFormat, options.zoneId, true)
164+
.parse(stringValue))
165165
.getOrElse {
166166
// If it fails to parse, then tries the way used in 2.0 and 1.x for backwards
167167
// compatibility.
168-
DateTimeUtils.stringToTime(stringValue).getTime * 1000L
168+
DateTimeUtils.stringToTimestamp(
169+
UTF8String.fromString(stringValue), options.zoneId).get
169170
}
170171
}
171172

@@ -175,28 +176,31 @@ class JacksonRecordParser(schema: DataType, val options: JSONOptions) extends Lo
175176

176177
case DateType =>
177178
(parser: JsonParser) =>
178-
parseJsonToken[java.lang.Integer](parser, dataType) {
179+
parseJsonToken[java.lang.Long](parser, dataType) {
179180
case VALUE_STRING =>
180181
val stringValue = parser.getText
181182
// This one will lose microseconds parts.
182183
// See https://issues.apache.org/jira/browse/SPARK-10681.x
183-
Int.box {
184-
Try(DateTimeUtils.millisToDays(options.dateFormat.parse(stringValue).getTime))
184+
Long.box {
185+
Try(TimestampFormatter(options.timestampFormat,
186+
options.zoneId, true).parse(stringValue))
185187
.orElse {
186188
// If it fails to parse, then tries the way used in 2.0 and 1.x for backwards
187189
// compatibility.
188-
Try(DateTimeUtils.millisToDays(DateTimeUtils.stringToTime(stringValue).getTime))
190+
Try(DateTimeUtils.stringToTimestamp(UTF8String.fromString(stringValue),
191+
options.zoneId).get)
189192
}
190193
.getOrElse {
191194
// In Spark 1.5.0, we store the data as number of days since epoch in string.
192195
// So, we just convert it to Int.
193-
stringValue.toInt
196+
stringValue.toLong
194197
}
195198
}
196199
}
197200

198201
case BinaryType =>
199202
(parser: JsonParser) =>
203+
200204
parseJsonToken[Array[Byte]](parser, dataType) {
201205
case VALUE_STRING => parser.getBinaryValue
202206
}

‎src/main/scala/org/apache/spark/sql/pulsar/PulsarConfigurationUtils.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ object PulsarConfigurationUtils {
2727
val fields = typeOf[T].members.collect{ case s: TermSymbol => s }.
2828
filter(s => s.isVal || s.isVar)
2929

30-
// then only keep the ones with a JsonIgnore annotation
31-
val ignores = fields.flatMap(f => f.annotations.find(_.tpe =:= typeOf[JsonIgnore]).
30+
// then only keep the ones without a JsonIgnore annotation
31+
val ignores = fields.flatMap(f => f.annotations.find(_.tree.tpe =:= typeOf[JsonIgnore]).
3232
map((f, _))).map(t => t._1).toList
3333

3434
fields.filterNot(ignores.contains).map(_.name.toString)

0 commit comments

Comments
 (0)