Merge pull request #793 from LongLonger/elastic6.1.1_zhongshu_dev_01

Author: zhongshu

.gitignore

@@ -20,3 +20,4 @@ src/_site/vendor/codemirror/mode/jinja2/.goutputstream-*
 src/_site/node_modules
 src/site-server/node_modules
 *~
+.DS_Store

README.md

@@ -637,7 +637,9 @@ http://localhost:9200/_sql/_explain?sql=select * from indexName limit 10
 *  SQL Select
 *  SQL Delete
 *  SQL Where
+    * can use "case when" in where clause
 *  SQL Order By
+    * can use "case when" in order by clause
 *  SQL Group By
 *  SQL AND & OR
 *  SQL Like
@@ -663,9 +665,20 @@ http://localhost:9200/_sql/_explain?sql=select * from indexName limit 10
 *  SQL log10
 *  SQL substring
 *  SQL round
+    * eg: round(10.135, 2) --> 10.14
 *  SQL sqrt
 *  SQL concat_ws
 *  SQL union and minus
+*  SQL case when
+    * can use "in"、"not in" judge in case when clause
+    * can execute simple calculation in case when clause, eg : case when 1 = 1 then field_1 + field_2 else 0 end
+*  SQL if
+    * select if(sex='1','男','女') from t_user;
+*  SQL limit
+    * can set aggregation bucket size and shard size by setting limit, shardSize = 20 * bucketSize
+    * eg: select city,count(*) as user_count from t_user group by city limit 100;
+    * on the above example, the bucket size is 100, shard size is 20*100 = 2000
+    
 
 ## JDBC Support (Experimental feature)
 

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.nlpcn</groupId>
 	<artifactId>elasticsearch-sql</artifactId>
-	<version>6.5.3.0</version>
+	<version>6.5.3.1</version>
 	<packaging>jar</packaging>
 	<description>Query elasticsearch using SQL</description>
 	<name>elasticsearch-sql</name>

src/_site/controllers.js

@@ -180,6 +180,7 @@ function updateWithScrollIfNeeded (query) {
     if(selectedQuery != "" && selectedQuery != undefined){
       query = selectedQuery;
     }
+
     query = updateWithScrollIfNeeded(query);
 		$http.post($scope.url + "_sql", query)
 		.success(function(data, status, headers, config) {

src/_site/query.js

@@ -222,12 +222,12 @@ var AggregationQueryResultHandler = function(data) {
             }
         }
 
-        else {
+        else { //zhongshu-comment 没有子bucket了，就是最里面的那一层了
             var obj = $.extend({}, additionalColumns)
             if(bucketName != undefined) {
                 if(bucketName != undefined) {
                     if("key_as_string" in bucket){
-                        obj[bucketName] = bucket["key_as_string"]
+                        obj[bucketName] = bucket["key_as_string"] //zhongshu-comment 给字段取别名
                     }
                     else {
                         obj[bucketName] = bucket.key
@@ -238,7 +238,7 @@ var AggregationQueryResultHandler = function(data) {
             for(var field in bucket) {
 
                 var bucketValue = bucket[field]
-                if(bucketValue.buckets != undefined ){
+                if(bucketValue.buckets != undefined ){ //zhongshu-comment 如果还有子bucket的话，那就继续递归
                     var newRows = getRows(subBucketName, bucketValue, newAdditionalColumns);
                     $.merge(rows, newRows);
                     continue;
@@ -272,7 +272,7 @@ var AggregationQueryResultHandler = function(data) {
         return rows
     }
 
-
+    //zhongshu-comment 递归
     function fillFieldsForSpecificAggregation(obj,value,field)
     {   
 
@@ -287,6 +287,7 @@ var AggregationQueryResultHandler = function(data) {
         return;
     }
 
+    //zhongshu-comment 递归
     function getSubBuckets(bucket) {
         var subBuckets = [];
         for(var field in bucket) {
@@ -297,7 +298,7 @@ var AggregationQueryResultHandler = function(data) {
                 }
             }
             else {
-                innerAgg = bucket[field];
+                innerAgg = bucket[field]; //zhongshu-comment innerAgg这个变量是哪来的，貌似没声明，到时问问松哥
                 for(var innerField in innerAgg){
                     if(typeof(innerAgg[innerField])=="object"){
                         innerBuckets = getSubBuckets(innerAgg[innerField]);
@@ -312,7 +313,7 @@ var AggregationQueryResultHandler = function(data) {
 
 
     this.data = data
-    this.flattenBuckets = getRows(undefined, data.aggregations, {})
+    this.flattenBuckets = getRows(undefined, data.aggregations, {}) //zhongshu-comment 入口
 };
 
 AggregationQueryResultHandler.prototype.getHead = function() {

src/main/java/org/elasticsearch/plugin/nlpcn/RestSqlAction.java

@@ -49,28 +49,34 @@ protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient cli
             sql = request.content().utf8ToString();
         }
         try {
-        SearchDao searchDao = new SearchDao(client);
-        QueryAction queryAction= null;
+            SearchDao searchDao = new SearchDao(client);
+            QueryAction queryAction = null;
 
-            queryAction = searchDao.explain(sql);
+            queryAction = searchDao.explain(sql);//zhongshu-comment 语法解析，将sql字符串解析为一个Java查询对象
 
-        // TODO add unittests to explain. (rest level?)
-        if (request.path().endsWith("/_explain")) {
-            final String jsonExplanation = queryAction.explain().explain();
-            return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, XContentType.JSON.mediaType(), jsonExplanation));
-        } else {
-            Map<String, String> params = request.params();
-            RestExecutor restExecutor = ActionRequestRestExecuterFactory.createExecutor(params.get("format"));
-            final QueryAction finalQueryAction = queryAction;
-            //doing this hack because elasticsearch throws exception for un-consumed props
-            Map<String,String> additionalParams = new HashMap<>();
-            for (String paramName : responseParams()) {
-                if (request.hasParam(paramName)) {
-                    additionalParams.put(paramName, request.param(paramName));
+            // TODO add unit tests to explain. (rest level?)
+            if (request.path().endsWith("/_explain")) {
+                final String jsonExplanation = queryAction.explain().explain();
+                return channel -> channel.sendResponse(new BytesRestResponse(RestStatus.OK, XContentType.JSON.mediaType(), jsonExplanation));
+            } else {
+                Map<String, String> params = request.params();
+
+                //zhongshu-comment 生成一个负责用rest方式查询es的对象RestExecutor，返回的实现类是：ElasticDefaultRestExecutor
+                RestExecutor restExecutor = ActionRequestRestExecuterFactory.createExecutor(params.get("format"));
+                final QueryAction finalQueryAction = queryAction;
+                //doing this hack because elasticsearch throws exception for un-consumed props
+                Map<String, String> additionalParams = new HashMap<>();
+                for (String paramName : responseParams()) {
+                    if (request.hasParam(paramName)) {
+                        additionalParams.put(paramName, request.param(paramName));
+                    }
                 }
+                //zhongshu-comment restExecutor.execute()方法里会调用es查询的相关rest api
+                //zhongshu-comment restExecutor.execute()方法的第1、4个参数是框架传进来的参数，第2、3个参数是可以自己生成的参数，所以要多注重一点
+                //zhongshu-comment 默认调用的是ElasticDefaultRestExecutor这个子类
+                //todo 这是什么语法：搜索java8 -> lambda表达式：https://blog.csdn.net/ioriogami/article/details/12782141
+                return channel -> restExecutor.execute(client, additionalParams, finalQueryAction, channel);
             }
-            return channel -> restExecutor.execute(client,additionalParams, finalQueryAction,channel);
-        }
         } catch (SqlParseException | SQLFeatureNotSupportedException e) {
             e.printStackTrace();
         }

src/main/java/org/elasticsearch/plugin/nlpcn/executors/ElasticDefaultRestExecutor.java

@@ -33,25 +33,37 @@ public ElasticDefaultRestExecutor() {
 
     /**
      * Execute the ActionRequest and returns the REST response using the channel.
+     * zhongshu-comment 第二个参数Map<String, String> params 并没有被使用
      */
     @Override
     public void execute(Client client, Map<String, String> params, QueryAction queryAction, RestChannel channel) throws Exception {
+        //zhongshu-comment queryAction的使命结束了，交由SqlElasticRequestBuilder接力，SqlElasticRequestBuilder是es-sql自己定义的一个类，不是es原生api
         SqlElasticRequestBuilder requestBuilder = queryAction.explain();
+
+        //zhongshu-comment 看这行，将QueryAction对象转为es查询对象，这个是重点了，到这步就已经成功将sql字符串转化为es查询请求了
+        //zhongshu-comment ActionRequest是es的原生api
         ActionRequest request = requestBuilder.request();
 
-        if (requestBuilder instanceof JoinRequestBuilder) {
+        //zhongshu-comment 应该是分别对应6中QueryAction子类实现
+        if (requestBuilder instanceof JoinRequestBuilder) { //zhongshu-comment 对应连接查询：ESJoinQueryAction
             ElasticJoinExecutor executor = ElasticJoinExecutor.createJoinExecutor(client, requestBuilder);
             executor.run();
             executor.sendResponse(channel);
-        } else if (requestBuilder instanceof MultiQueryRequestBuilder) {
+        } else if (requestBuilder instanceof MultiQueryRequestBuilder) { //zhongshu-comment 对应union查询：MultiQueryAction
             ElasticHitsExecutor executor = MultiRequestExecutorFactory.createExecutor(client, (MultiQueryRequestBuilder) requestBuilder);
             executor.run();
             sendDefaultResponse(executor.getHits(), channel);
         } else if (request instanceof SearchRequest) {
+            //zhongshu-comment 对应的QueryAction实现子类：DefaultQueryAction、AggregationQueryAction
+            //zhongshu-comment 对应的SqlElasticRequestBuilder实现子类：SqlElasticSearchRequestBuilder
             client.search((SearchRequest) request, new RestStatusToXContentListener<>(channel));
         } else if (request instanceof DeleteByQueryRequest) {
+            //zhongshu-comment 对应的QueryAction实现子类：DeleteQueryAction
+            //zhongshu-comment 对应的SqlElasticRequestBuilder实现子类：SqlElasticDeleteByQueryRequestBuilder
             requestBuilder.getBuilder().execute(new BulkIndexByScrollResponseContentListener(channel, Maps.newHashMap()));
         } else if (request instanceof GetIndexRequest) {
+            //zhongshu-comment 对应的QueryAction实现子类：ShowQueryAction
+            //zhongshu-comment 对应的SqlElasticRequestBuilder实现子类：是一个匿名内部类，跳进去queryAction.explain()看
             requestBuilder.getBuilder().execute(new GetIndexRequestRestListener(channel, (GetIndexRequest) request));
         } else if (request instanceof SearchScrollRequest) {
             client.searchScroll((SearchScrollRequest) request, new RestStatusToXContentListener<>(channel));

src/main/java/org/nlpcn/es4sql/SQLFunctions.java

@@ -25,13 +25,34 @@ public class SQLFunctions {
             "random", "abs", //nummber operator
             "split", "concat_ws", "substring", "trim",//string operator
             "add", "multiply", "divide", "subtract", "modulus",//binary operator
-            "field", "date_format"
+            "field", "date_format", "if"
     );
 
 
     public static Tuple<String, String> function(String methodName, List<KVValue> paramers, String name,boolean returnValue) {
         Tuple<String, String> functionStr = null;
-        switch (methodName) {
+        switch (methodName.toLowerCase()) {
+            case "if":
+                String nameIF = "";
+                String caseString = "";
+                if(paramers.get(0).value instanceof SQLInListExpr){
+                    nameIF += methodName+"("+((SQLInListExpr) paramers.get(0).value).getExpr()+" in (";
+                    String left = "doc['"+((SQLInListExpr) paramers.get(0).value).getExpr().toString()+"'].value";
+                    List<SQLExpr> targetList = ((SQLInListExpr) paramers.get(0).value).getTargetList();
+                    for(SQLExpr a:targetList){
+                        caseString += left + " == '" + a.toString() + "' ||";
+                        nameIF += a.toString()+",";
+                    }
+                    caseString = caseString.substring(0,caseString.length()-2);
+                    nameIF = nameIF.substring(0,nameIF.length()-1)+"),";
+                }else{
+                    String left = "doc['"+paramers.get(0).key+"'].value";
+                    caseString += left + " == '" + paramers.get(0).value+"'";
+                    nameIF = methodName+"("+paramers.get(0).toString()+",";
+                }
+                nameIF += paramers.get(1).value+","+paramers.get(2).value+")";
+                functionStr = new Tuple<>(nameIF,"if(("+caseString+")){"+paramers.get(1).value+"} else {"+paramers.get(2).value+"}");
+                break;
             case "split":
                 if (paramers.size() == 3) {
                     functionStr = split(Util.expr2Object((SQLExpr) paramers.get(0).value).toString(),
@@ -66,6 +87,12 @@ public static Tuple<String, String> function(String methodName, List<KVValue> pa
             case "abs":
             case "round":
             case "floor":
+                if (paramers.size() == 2) {
+                    //zhongshu-comment es的round()默认是保留到个位，这里给round()函数加上精确到小数点后第几位的功能
+                    int decimalPrecision = Integer.parseInt(paramers.get(1).value.toString());
+                    functionStr = mathRoundTemplate("Math."+methodName,methodName,Util.expr2Object((SQLExpr) paramers.get(0).value).toString(), name, decimalPrecision);
+                    break;
+                }
             case "ceil":
             case "cbrt":
             case "rint":
@@ -131,7 +158,7 @@ public static Tuple<String, String> function(String methodName, List<KVValue> pa
             default:
 
         }
-        if(returnValue){
+        if(returnValue && !methodName.equalsIgnoreCase("if")){
             String generatedFieldName = functionStr.v1();
             String returnCommand = ";return " + generatedFieldName +";" ;
             String newScript = functionStr.v2() + returnCommand;
@@ -328,6 +355,23 @@ private static Tuple<String, String> mathSingleValueTemplate(String methodName,
 
     }
 
+    private static Tuple<String, String> mathRoundTemplate(String methodName, String fieldName, String strColumn, String valueName, int decimalPrecision) {
+
+        StringBuilder sb = new StringBuilder("1");
+        for (int i = 0; i < decimalPrecision; i++) {
+            sb.append("0");
+        }
+        double num = Double.parseDouble(sb.toString());
+
+        String name = fieldName + "_" + random();
+        if (valueName == null) {
+            return new Tuple<>(name, "def " + name + " = " + methodName + "((doc['" + strColumn + "'].value) * " + num + ")/" + num);
+        } else {
+            return new Tuple<>(name, strColumn + ";def " + name + " = " + methodName + "((" + valueName + ") * " + num + ")/" + num);
+        }
+
+    }
+
     public static Tuple<String, String> strSingleValueTemplate(String methodName, String strColumn, String valueName) {
         String name = methodName + "_" + random();
         if (valueName == null) {

src/main/java/org/nlpcn/es4sql/Util.java

@@ -101,8 +101,14 @@ public static Object getScriptValueWithQuote(SQLExpr expr, String quote) throws
             return ((SQLNumericLiteralExpr) expr).getNumber();
         } else if (expr instanceof SQLNullExpr) {
             return ((SQLNullExpr) expr).toString().toLowerCase();
-        }
-        throw new SqlParseException("could not parse sqlBinaryOpExpr need to be identifier/valuable got" + expr.getClass().toString() + " with value:" + expr.toString());
+        } else if (expr instanceof  SQLBinaryOpExpr) {
+            //zhongshu-comment 该分支由忠树添加
+            String left = "doc['" + ((SQLBinaryOpExpr) expr).getLeft().toString() + "'].value";
+            String operator = ((SQLBinaryOpExpr) expr).getOperator().getName();
+            String right = "doc['" + ((SQLBinaryOpExpr) expr).getRight().toString() + "'].value";
+            return left + operator + right;
+        }
+        throw new SqlParseException("could not parse sqlBinaryOpExpr need to be identifier/valuable got " + expr.getClass().toString() + " with value:" + expr.toString());
     }
 
     public static boolean isFromJoinOrUnionTable(SQLExpr expr) {
@@ -139,7 +145,7 @@ public static double[] KV2DoubleArr(List<KVValue> params) {
         double[] ds = new double[params.size()];
         int i = 0;
         for (KVValue v : params) {
-            ds[i] = Double.parseDouble(v.value.toString());
+            ds[i] = ((Number) v.value).doubleValue();
             i++;
         }
         return ds;

src/main/java/org/nlpcn/es4sql/domain/Condition.java

@@ -89,6 +89,10 @@ public SQLExpr getValueExpr() {
     private boolean isChildren;
     private String childType;
 
+    public Condition(CONN conn) {
+        super(conn);
+    }
+
     public Condition(CONN conn, String field, SQLExpr nameExpr, String condition, Object obj, SQLExpr valueExpr) throws SqlParseException {
         this(conn, field, nameExpr, condition, obj, valueExpr, null);
     }