read lzo

Author: imperio-wxm

hadoop-file-type/pom.xml

@@ -42,7 +42,7 @@
                             <transformers>
                                 <transformer
                                         implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>com.wxmimperio.hadoop.GetFileOrPathSize</mainClass>
+                                    <mainClass>com.wxmimperio.hadoop.ReadLzoFile</mainClass>
                                 </transformer>
                             </transformers>
                         </configuration>
@@ -66,16 +66,15 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.test.skip>true</maven.test.skip>
-        <hadoop.version>2.6.0-cdh5.11.1</hadoop.version>
+        <hadoop.version>2.8.4</hadoop.version>
         <hadoop.scope>compile</hadoop.scope>
     </properties>
 
     <dependencies>
         <dependency>
-            <groupId>com.hadoop.lzo</groupId>
+            <groupId>com.hadoop.compression</groupId>
             <artifactId>hadoop-lzo</artifactId>
-            <version>0.4.15</version>
-            <scope>${hadoop.scope}</scope>
+            <version>0.4.20</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -106,21 +105,4 @@
             <scope>test</scope>
         </dependency>
     </dependencies>
-
-    <repositories>
-        <repository>
-            <id>nexus-public</id>
-            <url>http://nexus.dw.sdo.com/content/groups/public/</url>
-        </repository>
-
-        <repository>
-            <id>dw-releases</id>
-            <url>http://nexus.dw.sdo.com/content/repositories/dw-releases/</url>
-        </repository>
-
-        <repository>
-            <id>dw-snapshots</id>
-            <url>http://nexus.dw.sdo.com/content/repositories/dw-snapshots/</url>
-        </repository>
-    </repositories>
 </project>

hadoop-file-type/src/main/java/com/wxmimperio/hadoop/ReadLzoFile.java

@@ -0,0 +1,56 @@
+package com.wxmimperio.hadoop;
+
+import com.hadoop.compression.lzo.LzopCodec;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.*;
+import java.util.Arrays;
+
+/**
+ * Created with IntelliJ IDEA.
+ *
+ * @author weiximing
+ * @version 1.0.0
+ * @className ReadLzoFile.java
+ * @description This is the description of ReadLzoFile.java
+ * @createTime 2021-01-27 17:07:00
+ */
+public class ReadLzoFile {
+    private final static String ORC = ".orc";
+    private final static String LZO = ".lzo";
+
+    public static void main(String[] args) throws Exception {
+
+        String delimiter = "\t";
+
+        String localPath = args[0];
+        File[] localFiles = new File(localPath).listFiles();
+        for (File localFile : localFiles) {
+            if (localFile.isHidden()) {
+                continue;
+            }
+            try (InputStream inputStream = wrapInputStream(new FileInputStream(localFile), localFile.getName());
+                 BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) {
+                String line;
+                while ((line = br.readLine()) != null) {
+                    String[] fields = StringUtils.splitByWholeSeparatorPreserveAllTokens(line, delimiter);
+                    System.out.println(Arrays.toString(fields));
+                }
+            }
+        }
+    }
+
+    private static InputStream wrapInputStream(InputStream origin, String fileName) throws IOException {
+        if (StringUtils.endsWithIgnoreCase(fileName, LZO)) {
+            LzopCodec codec = new LzopCodec();
+            codec.setConf(new Configuration());
+            return codec.createInputStream(origin);
+        }
+        if (StringUtils.endsWithIgnoreCase(fileName, ORC)) {
+            // todo
+            return origin;
+        }
+        return origin;
+    }
+}