function.xml is perceivable (#390)

* function.xml is perceivable

* fix bin/startup.sh

* Optimize the definition of function

---------

Co-authored-by: spricoder <[email protected]>

configuration/benchmark.bat

@@ -84,7 +84,7 @@ goto :eof
 
 rem echo CLASSPATH: %CLASSPATH%
 
-"%JAVA_HOME%\bin\java" %JAVA_OPTS% -cp .;./lib/* %MAIN_CLASS% -cf %BENCHMARK_HOME%/conf/config.properties
+"%JAVA_HOME%\bin\java" %JAVA_OPTS% -cp .;./lib/* %MAIN_CLASS% -cf %BENCHMARK_HOME%/conf
 goto finally
 
 :err

configuration/bin/startup.sh

@@ -29,7 +29,7 @@ show_help() {
   echo " -cf          Specify configuration file."
   echo " -heapsize    Specify HEAP_SIZE."
   echo " -maxheapsize Specify MAX_HEAP_SIZE."
-  echo "example: ./benchmark.sh -cf conf/config.properties -heapsize 1G -maxheapsize 2G"
+  echo "example: ./benchmark.sh -cf conf -heapsize 1G -maxheapsize 2G"
 }
 
 while [[ $# -gt 0 ]]; do
@@ -80,20 +80,20 @@ fi
 
 # check BENCHMARK_HOME
 if [ -z "${BENCHMARK_HOME}" ]; then
-  export BENCHMARK_HOME="$(cd "`dirname "$0"`"/.. && pwd)"
+  export BENCHMARK_HOME="$(cd "$(dirname "$0")/.." && pwd)"
 fi
 
 # check $benchmark_conf
-if [ -z $benchmark_conf ] ; then
-  benchmark_conf=${BENCHMARK_HOME}/conf/config.properties
+if [ -z "${benchmark_conf}" ] ; then
+  benchmark_conf=${BENCHMARK_HOME}/conf
 else
   benchmark_conf="$(cd "$(dirname "$benchmark_conf")" && pwd)/$(basename "$benchmark_conf")"
   if [ ! -e "$benchmark_conf" ]; then
     echo "The file $benchmark_conf does not exist."
     exit 1
   fi
 fi
-echo Using configuration file: $benchmark_conf
+echo Using configuration file: "${benchmark_conf}"
 
 # set MAIN_CLASS
 MAIN_CLASS=cn.edu.tsinghua.iot.benchmark.App

configuration/conf/config.properties

@@ -47,7 +47,7 @@
 # generateDataMode      生成数据模式，生成Benchmark本身识别的数据
 # verificationWriteMode 单数据库正确性写入模式，需要配置 FILE_PATH 以及 DATA_SET
 # verificationQueryMode 单数据库正确性查询模式，需要配置 FILE_PATH 以及 DATA_SET
-# BENCHMARK_WORK_MODE=testWithDefaultPath
+BENCHMARK_WORK_MODE=generateDataMode
 
 # 对于数据写入或查询，限制最长耗时，设置为0表示无限制，单位为ms
 # 对数据写入，最大误差约为WRITE_OPERATION_TIMEOUT_MS
@@ -111,13 +111,13 @@
 
 ############## 数据：设备、传感器、客户端 ##################
 # 设备总数
-# DEVICE_NUMBER=6000
+DEVICE_NUMBER=10
 
 # 实际写入设备比例，(0, 1]
 # REAL_INSERT_RATE=1.0
 
 # 每个设备的传感器总数
-# SENSOR_NUMBER=200
+SENSOR_NUMBER=6
 
 # 各个传感器时间戳是否对齐
 # IS_SENSOR_TS_ALIGNMENT=true

core/src/main/java/cn/edu/tsinghua/iot/benchmark/App.java

@@ -46,7 +46,7 @@ public static void main(String[] args) throws SQLException {
             + "bytes.");
 
     if (args == null || args.length == 0) {
-      args = new String[] {"-cf", "configuration/conf/config.properties"};
+      args = new String[] {"-cf", "configuration/conf"};
     }
     CommandCli cli = new CommandCli();
     if (!cli.init(args)) {

core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Config.java

@@ -21,7 +21,6 @@
 
 import cn.edu.tsinghua.iot.benchmark.entity.Sensor;
 import cn.edu.tsinghua.iot.benchmark.entity.enums.SensorType;
-import cn.edu.tsinghua.iot.benchmark.function.Function;
 import cn.edu.tsinghua.iot.benchmark.function.FunctionParam;
 import cn.edu.tsinghua.iot.benchmark.function.FunctionXml;
 import cn.edu.tsinghua.iot.benchmark.mode.enums.BenchmarkMode;
@@ -35,11 +34,13 @@
 import javax.xml.bind.Unmarshaller;
 
 import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.*;
 import java.util.concurrent.atomic.AtomicLong;
 
 public class Config {
-  private static Logger LOGGER = LoggerFactory.getLogger(Config.class);
+  private static final Logger LOGGER = LoggerFactory.getLogger(Config.class);
 
   // 初始化
   // 初始化：清理数据
@@ -407,11 +408,18 @@ public class Config {
   /** Sensor function */
   private Map<String, FunctionParam> SENSOR_FUNCTION = new HashMap<>();
 
+  public String getHomeDir() {
+    // When start benchmark with the script, the environment variables will be set.
+    // But in developer mode it will return another dir to find resources.
+    return System.getProperty(Constants.BENCHMARK_HOME, null);
+  }
+
   /** init inner functions */
   public void initInnerFunction() {
     FunctionXml xml = null;
+    String configFolder = System.getProperty(Constants.BENCHMARK_CONF, "configuration/conf");
     try {
-      InputStream input = Function.class.getResourceAsStream("/function.xml");
+      InputStream input = Files.newInputStream(Paths.get(configFolder + "/function.xml"));
       JAXBContext context = JAXBContext.newInstance(FunctionXml.class, FunctionParam.class);
       Unmarshaller unmarshaller = context.createUnmarshaller();
       xml = (FunctionXml) unmarshaller.unmarshal(input);

core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/ConfigDescriptor.java

@@ -70,12 +70,12 @@ public Config getConfig() {
 
   /** load properties from config.properties */
   private void loadProps() {
-    String url =
-        System.getProperty(Constants.BENCHMARK_CONF, "configuration/conf/config.properties");
+    String url = System.getProperty(Constants.BENCHMARK_CONF, "configuration/conf");
     if (url != null) {
+      url += "/config.properties";
       InputStream inputStream;
       try {
-        inputStream = new FileInputStream(new File(url));
+        inputStream = new FileInputStream(url);
       } catch (FileNotFoundException e) {
         LOGGER.warn("Fail to find config file {}", url);
         return;

core/src/main/java/cn/edu/tsinghua/iot/benchmark/conf/Constants.java

@@ -27,6 +27,8 @@ public class Constants {
   public static final long START_TIMESTAMP =
       TimeUtils.convertDateStrToTimestamp(config.getSTART_TIME());
   public static final String CONSOLE_PREFIX = "iot-benchmark>";
+
+  public static final String BENCHMARK_HOME = "BENCHMARK_HOME";
   public static final String BENCHMARK_CONF = "benchmark-conf";
 
   public static final String MYSQL_DRIVENAME = "com.mysql.jdbc.Driver";

core/src/main/java/cn/edu/tsinghua/iot/benchmark/measurement/persistence/csv/CSVRecorder.java

@@ -35,7 +35,6 @@
 import java.io.IOException;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
-import java.sql.Date;
 import java.text.SimpleDateFormat;
 import java.util.Collections;
 import java.util.Map;
@@ -51,7 +50,6 @@ public class CSVRecorder extends TestDataPersistence {
   /** The count of write retry */
   private static final int WRITE_RETRY_COUNT = 5;
 
-  private static final int MAX_COMPRESS_TIME = 10 * 60 * 1000;
   /** reentrantLock used for writing result into file */
   private static final ReentrantLock reentrantLock = new ReentrantLock(true);
 
@@ -101,10 +99,8 @@ public CSVRecorder() {
     }
     localName = localName.replace("-", "_");
     localName = localName.replace(".", "_");
-    Date date = new Date(EXP_TIME);
-    SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy_MM_dd");
-    String day = dateFormat.format(date);
-    confDir = System.getProperty(Constants.BENCHMARK_CONF);
+
+    confDir = System.getProperty(Constants.BENCHMARK_CONF) + "/config.properties";
     dataDir = confDir.substring(0, confDir.length() - 23) + "/data";
     csvDir = dataDir + "/csv";
     File dataFile = new File(dataDir);

iginx/src/assembly/assembly.xml

@@ -40,6 +40,10 @@
             <directory>${maven.multiModuleProjectDirectory}/configuration/conf/</directory>
             <outputDirectory>conf</outputDirectory>
         </fileSet>
+        <fileSet>
+            <directory>${maven.multiModuleProjectDirectory}/core/src/main/resources</directory>
+            <outputDirectory>resources</outputDirectory>
+        </fileSet>
     </fileSets>
     <files>
         <file>