Skip to content

Commit

Permalink
PIG-5414 Fix the build on Linux ARM64
Browse files Browse the repository at this point in the history
Upgrade Apache Avro to 1.8.2 because otherwise TestAvroStorage and TestOrcStorage*.java fail.
Update Snappy because the old version does not provide native library for aarch64 (ARM64)
Add TravisCI build config that builds and tests on AMD64 and ARM64 architectures
  • Loading branch information
martin-g committed Oct 13, 2021
1 parent 59ec4a3 commit 2ee3588
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 7 deletions.
46 changes: 46 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

os: linux
dist: focal

arch:
- amd64
- arm64

install:
- lscpu
- ARCH=`uname -p`
- echo $ARCH
- JDK_X64="https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u272-b10/OpenJDK8U-jdk_x64_linux_hotspot_8u272b10.tar.gz"
- JDK_ARM64="https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u272-b10/OpenJDK8U-jdk_aarch64_linux_hotspot_8u272b10.tar.gz"
- if test "X$ARCH" = "Xaarch64"; then JDK_URL=$JDK_ARM64; else JDK_URL=$JDK_X64; fi
- wget -q $JDK_URL && tar xzf OpenJDK*.tar.gz
- mv jdk8* jdk
- export JAVA_HOME=`pwd`/jdk
- wget -q https://mirrors.netix.net/apache/ant/binaries/apache-ant-1.10.9-bin.tar.gz && tar xzf apache-ant-*-bin.tar.gz
- export ANT_HOME=`pwd`/apache-ant-1.10.9
- wget -q https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && tar xzf apache-maven*.tar.gz
- export M2_HOME=`pwd`/apache-maven-3.6.3
- export PATH="$JAVA_HOME/bin:$ANT_HOME/bin:$M2_HOME/bin:$PATH"
- java -version
- ant -version
- mvn -version


script:
- ant clean test-unit

4 changes: 2 additions & 2 deletions ivy.xml
Original file line number Diff line number Diff line change
Expand Up @@ -181,12 +181,12 @@
<dependency org="org.apache.avro" name="trevni-core" rev="${avro.version}"
conf="compile->default;checkstyle->master">
<artifact name="trevni-core" ext="jar" />
<artifact name="trevni-core" type="test-jar" ext="jar" m:classifier="tests"/>
<!-- <artifact name="trevni-core" type="test-jar" ext="jar" m:classifier="tests"/>-->
</dependency>
<dependency org="org.apache.avro" name="trevni-avro" rev="${avro.version}"
conf="compile->default;checkstyle->master">
<artifact name="trevni-avro" ext="jar" />
<artifact name="trevni-avro" type="test-jar" ext="jar" m:classifier="tests"/>
<!-- <artifact name="trevni-avro" type="test-jar" ext="jar" m:classifier="tests"/>-->
<exclude org="org.apache.hadoop" module="hadoop-core"/>
</dependency>
<dependency org="org.apache.avro" name="avro-tools" rev="${avro.version}"
Expand Down
6 changes: 3 additions & 3 deletions ivy/libraries.properties
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ accumulo15.version=1.5.0
apacheant.version=1.7.1
apacherat.version=0.8
automaton.version=1.11-8
avro.version=1.7.5
avro.version=1.8.2
basjes-httpdlog-pigloader.version=5.0
commons-beanutils.version=1.7.0
commons-cli.version=1.2
Expand Down Expand Up @@ -89,10 +89,10 @@ jsr311-api.version=1.1.1
mockito.version=1.8.4
jansi.version=1.9
asm.version=3.3.1
snappy-java.version=1.1.1.3
snappy-java.version=1.1.8.4
tez.version=0.7.0
parquet-pig-bundle.version=1.9.0
snappy.version=0.2
snappy.version=0.4
leveldbjni.version=1.8
curator.version=2.6.0
htrace.version=3.1.0-incubating
Expand Down
3 changes: 1 addition & 2 deletions src/org/apache/pig/impl/util/avro/AvroRecordWriter.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@

import static org.apache.avro.file.DataFileConstants.DEFAULT_SYNC_INTERVAL;
import static org.apache.avro.file.DataFileConstants.DEFLATE_CODEC;
import static org.apache.avro.mapred.AvroOutputFormat.DEFAULT_DEFLATE_LEVEL;
import static org.apache.avro.mapred.AvroOutputFormat.DEFLATE_LEVEL_KEY;
import static org.apache.avro.mapred.AvroOutputFormat.SYNC_INTERVAL_KEY;

Expand Down Expand Up @@ -73,7 +72,7 @@ static void configureDataFileWriter(DataFileWriter<GenericData.Record> writer,
JobConf job) throws UnsupportedEncodingException {
if (FileOutputFormat.getCompressOutput(job)) {
int level = job.getInt(DEFLATE_LEVEL_KEY,
DEFAULT_DEFLATE_LEVEL);
CodecFactory.DEFAULT_DEFLATE_LEVEL);
String codecName = job.get(AvroJob.OUTPUT_CODEC, DEFLATE_CODEC);
CodecFactory factory = codecName.equals(DEFLATE_CODEC)
? CodecFactory.deflateCodec(level)
Expand Down

0 comments on commit 2ee3588

Please sign in to comment.