Merge remote-tracking branch 'connect/10.2' into 10.2

storage/connect/global.h

 /***********************************************************************/
 /*  GLOBAL.H: Declaration file used by all CONNECT implementations.    */
 /*  (C) Copyright MariaDB Corporation Ab                 							 */
-/*  Author Olivier Bertrand                              1993-2018     */
+/*  Author Olivier Bertrand                              1993-2020     */
 /***********************************************************************/
 
 /***********************************************************************/
@@ -89,14 +89,10 @@ extern "C" {
 #define PAT_LOG       "log"
 
 #if defined(UNIX) || defined(LINUX) || defined(UNIV_LINUX)
-  /*********************************************************************/
-  /*  printf does not accept null pointer for %s target.               */
-  /*********************************************************************/
+  // printf does not accept null pointer for %s target
   #define SVP(S)  ((S) ? S : "<null>")
 #else
-  /*********************************************************************/
-  /*  printf accepts null pointer for %s target.                       */
-  /*********************************************************************/
+  //  printf accepts null pointer for %s target
   #define SVP(S)  S
 #endif
 
@@ -112,9 +108,6 @@ extern "C" {
 /***********************************************************************/
 #include "os.h"
 
-typedef uint  OFFSET;
-typedef char  NAME[9];
-
 typedef struct {
   ushort Length;
   char   String[2];
@@ -127,6 +120,7 @@ typedef struct _global   *PGLOBAL;
 typedef struct _globplg  *PGS;
 typedef struct _activity *PACTIVITY;
 typedef struct _parm     *PPARM;
+typedef char   NAME[9];
 
 /***********************************************************************/
 /* Segment Sub-Allocation block structure declares.                    */
@@ -135,8 +129,8 @@ typedef struct _parm     *PPARM;
 /* restore them if needed. This scheme implies that no SubFree be used */
 /***********************************************************************/
 typedef struct {               /* Plug Area SubAlloc header            */
-  OFFSET To_Free;              /* Offset of next free block            */
-  uint   FreeBlk;              /* Size of remaining free memory        */
+  size_t To_Free;              /* Offset of next free block            */
+  size_t FreeBlk;              /* Size of remaining free memory        */
   } POOLHEADER, *PPOOLHEADER;
 
 /***********************************************************************/
@@ -188,11 +182,12 @@ typedef struct _parm {
 /***********************************************************************/
 typedef struct _global {            /* Global structure                */
   void     *Sarea;                  /* Points to work area             */
-  uint      Sarea_Size;             /* Work area size                  */
+  size_t    Sarea_Size;             /* Work area size                  */
 	PACTIVITY Activityp;
-  char      Message[MAX_STR];
+  char      Message[MAX_STR];				/* Message (result, error, trace)  */
 	ulong     More;										/* Used by jsonudf                 */
-	int       Createas;               /* To pass multi to ext tables     */
+	size_t    Saved_Size;             /* Saved work area to_free         */
+	bool      Createas;               /* To pass multi to ext tables     */
   void     *Xchk;                   /* indexes in create/alter         */
   short     Alchecked;              /* Checked for ALTER               */
   short     Mrr;                    /* True when doing mrr             */
@@ -212,17 +207,16 @@ DllExport char   *PlugGetMessage(PGLOBAL, int);
 #if defined(__WIN__)
 DllExport short   GetLineLength(PGLOBAL);   // Console line length
 #endif   // __WIN__
-DllExport PGLOBAL PlugInit(LPCSTR, uint);  // Plug global initialization
+DllExport PGLOBAL PlugInit(LPCSTR, size_t); // Plug global initialization
 DllExport int     PlugExit(PGLOBAL);        // Plug global termination
 DllExport LPSTR   PlugRemoveType(LPSTR, LPCSTR);
 DllExport LPCSTR  PlugSetPath(LPSTR to, LPCSTR prefix, LPCSTR name, LPCSTR dir);
 DllExport BOOL    PlugIsAbsolutePath(LPCSTR path);
-DllExport bool    AllocSarea(PGLOBAL, uint);
+DllExport bool    AllocSarea(PGLOBAL, size_t);
 DllExport void    FreeSarea(PGLOBAL);
-DllExport BOOL    PlugSubSet(void *, uint);
+DllExport BOOL    PlugSubSet(void *, size_t);
 DllExport void   *PlugSubAlloc(PGLOBAL, void *, size_t);
 DllExport char   *PlugDup(PGLOBAL g, const char *str);
-DllExport void   *MakePtr(void *, OFFSET);
 DllExport void    htrc(char const *fmt, ...);
 DllExport void    xtrc(uint, char const* fmt, ...);
 DllExport uint    GetTraceValue(void);
@@ -232,8 +226,24 @@ DllExport uint    GetTraceValue(void);
 #endif
 
 /***********************************************************************/
-/*  Non exported routine declarations.                                 */
+/*  Inline routine definitions.                                        */
+/***********************************************************************/
+/***********************************************************************/
+/* This routine makes a pointer from an offset to a memory pointer.    */
+/***********************************************************************/
+inline void* MakePtr(void* memp, size_t offset) {
+	// return ((offset == 0) ? NULL : &((char*)memp)[offset]);
+	return (!offset) ? NULL : (char *)memp + offset;
+} /* end of MakePtr */
+
+/***********************************************************************/
+/* This routine makes an offset from a pointer new format.             */
 /***********************************************************************/
-//void *PlugSubAlloc(PGLOBAL, void *, size_t);	 // Does throw
+inline size_t MakeOff(void* memp, void* ptr) {
+#if defined(_DEBUG)
+	assert(ptr > memp);
+#endif   // _DEBUG
+	return ((!ptr) ? 0 : (size_t)((char*)ptr - (size_t)memp));
+} /* end of MakeOff */
 
 /*-------------------------- End of Global.H --------------------------*/

storage/connect/ha_connect.cc

@@ -170,9 +170,9 @@
 #define JSONMAX      10             // JSON Default max grp size
 
 extern "C" {
-       char version[]= "Version 1.07.0001 November 12, 2019";
+       char version[]= "Version 1.07.0002 October 18, 2020";
 #if defined(__WIN__)
-       char compver[]= "Version 1.07.0001 " __DATE__ " "  __TIME__;
+       char compver[]= "Version 1.07.0002 " __DATE__ " "  __TIME__;
        char slash= '\\';
 #else   // !__WIN__
        char slash= '/';
@@ -251,11 +251,13 @@ bool    ExactInfo(void);
 USETEMP UseTemp(void);
 int     GetConvSize(void);
 TYPCONV GetTypeConv(void);
+bool    JsonAllPath(void);
 char   *GetJsonNull(void);
+int GetDefaultDepth(void);
 uint    GetJsonGrpSize(void);
 char   *GetJavaWrapper(void);
-uint    GetWorkSize(void);
-void    SetWorkSize(uint);
+size_t  GetWorkSize(void);
+void    SetWorkSize(size_t);
 extern "C" const char *msglang(void);
 
 static char *strz(PGLOBAL g, LEX_STRING &ls);
@@ -347,11 +349,19 @@ static MYSQL_THDVAR_ENUM(
   1,                               // def (AUTO)
   &usetemp_typelib);               // typelib
 
+#ifdef _WIN64
 // Size used for g->Sarea_Size
-static MYSQL_THDVAR_UINT(work_size,
+static MYSQL_THDVAR_ULONGLONG(work_size,
 	PLUGIN_VAR_RQCMDARG,
 	"Size of the CONNECT work area.",
-       NULL, NULL, SZWORK, SZWMIN, UINT_MAX, 1);
+	NULL, NULL, SZWORK, SZWMIN, ULONGLONG_MAX, 1);
+#else
+// Size used for g->Sarea_Size
+static MYSQL_THDVAR_ULONG(work_size,
+  PLUGIN_VAR_RQCMDARG, 
+  "Size of the CONNECT work area.",
+  NULL, NULL, SZWORK, SZWMIN, ULONG_MAX, 1);
+#endif
 
 // Size used when converting TEXT columns to VARCHAR
 static MYSQL_THDVAR_INT(conv_size,
@@ -386,6 +396,11 @@ static MYSQL_THDVAR_ENUM(
   1,                               // def (yes)
   &xconv_typelib);                 // typelib
 
+// Adding JPATH to all Json table columns
+static MYSQL_THDVAR_BOOL(json_all_path, PLUGIN_VAR_RQCMDARG,
+	"Adding JPATH to all Json table columns",
+	NULL, NULL, 0);							// NO by default
+
 // Null representation for JSON values
 static MYSQL_THDVAR_STR(json_null,
 	PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_MEMALLOC,
@@ -393,6 +408,12 @@ static MYSQL_THDVAR_STR(json_null,
 	//     check_json_null, update_json_null,
 	NULL, NULL, "<null>");
 
+// Default Json, XML or Mongo depth
+static MYSQL_THDVAR_INT(default_depth,
+	PLUGIN_VAR_RQCMDARG,
+	"Default depth used by Json, XML and Mongo discovery",
+	NULL, NULL, 0, -1, 16, 1);
+
 // Estimate max number of rows for JSON aggregate functions
 static MYSQL_THDVAR_UINT(json_grp_size,
        PLUGIN_VAR_RQCMDARG,             // opt
@@ -454,15 +475,17 @@ uint GetTraceValue(void)
 	{return (uint)(connect_hton ? THDVAR(current_thd, xtrace) : 0);}
 bool ExactInfo(void) {return THDVAR(current_thd, exact_info);}
 static bool CondPushEnabled(void) {return THDVAR(current_thd, cond_push);}
+bool JsonAllPath(void) {return THDVAR(current_thd, json_all_path);}
 USETEMP UseTemp(void) {return (USETEMP)THDVAR(current_thd, use_tempfile);}
 int GetConvSize(void) {return THDVAR(current_thd, conv_size);}
 TYPCONV GetTypeConv(void) {return (TYPCONV)THDVAR(current_thd, type_conv);}
 char *GetJsonNull(void)
 	{return connect_hton ? THDVAR(current_thd, json_null) : NULL;}
+int GetDefaultDepth(void) {return THDVAR(current_thd, default_depth);}
 uint GetJsonGrpSize(void)
   {return connect_hton ? THDVAR(current_thd, json_grp_size) : 10;}
-uint GetWorkSize(void) {return THDVAR(current_thd, work_size);}
-void SetWorkSize(uint) 
+size_t GetWorkSize(void) {return (size_t)THDVAR(current_thd, work_size);}
+void SetWorkSize(size_t) 
 {
   // Changing the session variable value seems to be impossible here
   // and should be done in a check function 
@@ -472,7 +495,8 @@ void SetWorkSize(uint)
 
 #if defined(JAVA_SUPPORT)
 char *GetJavaWrapper(void)
-{return connect_hton ? THDVAR(current_thd, java_wrapper) : (char*)"wrappers/JdbcInterface";}
+{return connect_hton ? THDVAR(current_thd, java_wrapper)
+	                   : (char*)"wrappers/JdbcInterface";}
 #endif   // JAVA_SUPPORT
 
 #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT)
@@ -621,6 +645,8 @@ ha_create_table_option connect_field_option_list[]=
   HA_FOPTION_NUMBER("FIELD_LENGTH", fldlen, 0, 0, INT_MAX32, 1),
   HA_FOPTION_STRING("DATE_FORMAT", dateformat),
   HA_FOPTION_STRING("FIELD_FORMAT", fieldformat),
+  HA_FOPTION_STRING("JPATH", jsonpath),
+	HA_FOPTION_STRING("XPATH", xmlpath),
 	HA_FOPTION_STRING("SPECIAL", special),
 	HA_FOPTION_ENUM("DISTRIB", opt, "scattered,clustered,sorted", 0),
   HA_FOPTION_END
@@ -1313,9 +1339,10 @@ int GetIntegerTableOption(PGLOBAL g, PTOS options, PCSZ opname, int idef)
   if ((ulonglong) opval == (ulonglong)NO_IVAL) {
 		PCSZ pv;
 
-    if ((pv= GetListOption(g, opname, options->oplist)))
-      opval= CharToNumber((char*)pv, strlen(pv), ULONGLONG_MAX, true);
-    else
+		if ((pv = GetListOption(g, opname, options->oplist))) {
+			// opval = CharToNumber((char*)pv, strlen(pv), ULONGLONG_MAX, false);
+			return atoi(pv);
+		} else
       return idef;
 
     } // endif opval
@@ -1567,7 +1594,8 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf)
     pcf->Offset= (int)fop->offset;
     pcf->Freq= (int)fop->freq;
     pcf->Datefmt= (char*)fop->dateformat;
-    pcf->Fieldfmt= (char*)fop->fieldformat;
+		pcf->Fieldfmt = fop->fieldformat ? (char*)fop->fieldformat
+			: fop->jsonpath ? (char*)fop->jsonpath : (char*)fop->xmlpath;
 	} else {
     pcf->Offset= -1;
     pcf->Freq= 0;
@@ -1577,6 +1605,9 @@ void *ha_connect::GetColumnOption(PGLOBAL g, void *field, PCOLINFO pcf)
 
   chset= (char *)fp->charset()->name;
 
+	if (!strcmp(chset, "binary"))
+		v = 'B';		// Binary string
+
   switch (fp->type()) {
     case MYSQL_TYPE_BLOB:
     case MYSQL_TYPE_VARCHAR:
@@ -2213,7 +2244,7 @@ int ha_connect::MakeRecord(char *buf)
 					case TYPE_BIN:
 						p= value->GetCharValue();
 						charset= &my_charset_bin;
-						rc= fp->store(p, strlen(p), charset, CHECK_FIELD_WARN);
+						rc= fp->store(p, value->GetSize(), charset, CHECK_FIELD_WARN);
 						break;
           case TYPE_DOUBLE:
             rc= fp->store(value->GetFloatValue());
@@ -4972,7 +5003,7 @@ int ha_connect::check_stmt(PGLOBAL g, MODE newmode, bool cras)
 	} // endif CheckCleanup
 
   if (cras)
-    g->Createas= 1;  // To tell external tables of a multi-table command
+    g->Createas= true;  // To tell external tables of a multi-table command
 
 	if (trace(1))
 		htrc("Calling CntCheckDB db=%s cras=%d\n", GetDBName(NULL), cras);
@@ -5322,90 +5353,99 @@ static char *encode(PGLOBAL g, const char *cnm)
   @return
     Return 0 if ok
 */
-static bool add_field(String *sql, const char *field_name, int typ, int len,
-                      int dec, char *key, uint tm, const char *rem, char *dft,
-                      char *xtra, char *fmt, int flag, bool dbf, char v)
-{
-  char var= (len > 255) ? 'V' : v;
-  bool q, error= false;
-  const char *type= PLGtoMYSQLtype(typ, dbf, var);
+static bool add_field(String* sql, TABTYPE ttp, const char* field_name, int typ,
+	                    int len, int dec, char* key, uint tm, const char* rem,
+	                    char* dft, char* xtra, char* fmt, int flag, bool dbf, char v) {
+#if defined(DEVELOPMENT)
+	// Some client programs regard CHAR(36) as GUID
+	char var = (len > 255 || len == 36) ? 'V' : v;
+#else
+	char var = (len > 255) ? 'V' : v;
+#endif
+	bool q, error = false;
+	const char* type = PLGtoMYSQLtype(typ, dbf, var);
 
-  error|= sql->append('`');
-  error|= sql->append(field_name);
-  error|= sql->append("` ");
-  error|= sql->append(type);
+	error |= sql->append('`');
+	error |= sql->append(field_name);
+	error |= sql->append("` ");
+	error |= sql->append(type);
 
 	if (typ == TYPE_STRING ||
 		(len && typ != TYPE_DATE && (typ != TYPE_DOUBLE || dec >= 0))) {
-    error|= sql->append('(');
-    error|= sql->append_ulonglong(len);
+		error |= sql->append('(');
+		error |= sql->append_ulonglong(len);
 
 		if (typ == TYPE_DOUBLE) {
-      error|= sql->append(',');
+			error |= sql->append(',');
 			// dec must be < len and < 31
-      error|= sql->append_ulonglong(MY_MIN(dec, (MY_MIN(len, 31) - 1)));
+			error |= sql->append_ulonglong(MY_MIN(dec, (MY_MIN(len, 31) - 1)));
 		} else if (dec > 0 && !strcmp(type, "DECIMAL")) {
-      error|= sql->append(',');
+			error |= sql->append(',');
 			// dec must be < len
-      error|= sql->append_ulonglong(MY_MIN(dec, len - 1));
+			error |= sql->append_ulonglong(MY_MIN(dec, len - 1));
 		} // endif dec
 
-    error|= sql->append(')');
+		error |= sql->append(')');
 	} // endif len
 
 	if (v == 'U')
-    error|= sql->append(" UNSIGNED");
+		error |= sql->append(" UNSIGNED");
 	else if (v == 'Z')
-    error|= sql->append(" ZEROFILL");
+		error |= sql->append(" ZEROFILL");
 
 	if (key && *key) {
-    error|= sql->append(" ");
-    error|= sql->append(key);
+		error |= sql->append(" ");
+		error |= sql->append(key);
 	} // endif key
 
 	if (tm)
-    error|= sql->append(STRING_WITH_LEN(" NOT NULL"), system_charset_info);
+		error |= sql->append(STRING_WITH_LEN(" NOT NULL"), system_charset_info);
 
 	if (dft && *dft) {
-    error|= sql->append(" DEFAULT ");
+		error |= sql->append(" DEFAULT ");
 
 		if (typ == TYPE_DATE)
-      q= (strspn(dft, "0123456789 -:/") == strlen(dft));
+			q = (strspn(dft, "0123456789 -:/") == strlen(dft));
 		else
-      q= !IsTypeNum(typ);
+			q = !IsTypeNum(typ);
 
 		if (q) {
-      error|= sql->append("'");
-      error|= sql->append_for_single_quote(dft, strlen(dft));
-      error|= sql->append("'");
+			error |= sql->append("'");
+			error |= sql->append_for_single_quote(dft, strlen(dft));
+			error |= sql->append("'");
 		} else
-      error|= sql->append(dft);
+			error |= sql->append(dft);
 
 	} // endif dft
 
 	if (xtra && *xtra) {
-    error|= sql->append(" ");
-    error|= sql->append(xtra);
+		error |= sql->append(" ");
+		error |= sql->append(xtra);
 	} // endif rem
 
 	if (rem && *rem) {
-    error|= sql->append(" COMMENT '");
-    error|= sql->append_for_single_quote(rem, strlen(rem));
-    error|= sql->append("'");
+		error |= sql->append(" COMMENT '");
+		error |= sql->append_for_single_quote(rem, strlen(rem));
+		error |= sql->append("'");
 	} // endif rem
 
 	if (fmt && *fmt) {
-    error|= sql->append(" FIELD_FORMAT='");
-    error|= sql->append_for_single_quote(fmt, strlen(fmt));
-    error|= sql->append("'");
+		switch (ttp) {
+		case TAB_JSON: error |= sql->append(" JPATH='"); break;
+		case TAB_XML:  error |= sql->append(" XPATH='"); break;
+		default:	     error |= sql->append(" FIELD_FORMAT='");
+		} // endswitch ttp
+
+		error |= sql->append_for_single_quote(fmt, strlen(fmt));
+		error |= sql->append("'");
 	} // endif flag
 
 	if (flag) {
-    error|= sql->append(" FLAG=");
-    error|= sql->append_ulonglong(flag);
+		error |= sql->append(" FLAG=");
+		error |= sql->append_ulonglong(flag);
 	} // endif flag
 
-  error|= sql->append(',');
+	error |= sql->append(',');
 	return error;
 } // end of add_field
 
@@ -6028,7 +6068,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd,
 						len= 256;      // STRBLK's have 0 length
 
 					// Now add the field
-					if (add_field(&sql, cnm, typ, len, dec, NULL, tm,
+					if (add_field(&sql, ttp, cnm, typ, len, dec, NULL, tm,
 						NULL, NULL, NULL, NULL, flg, dbf, v))
 						rc= HA_ERR_OUT_OF_MEM;
 				} // endfor crp
@@ -6222,7 +6262,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd,
 							prec= 0;
 
 						// Now add the field
-						if (add_field(&sql, cnm, typ, prec, dec, key, tm, rem, dft, xtra,
+						if (add_field(&sql, ttp, cnm, typ, prec, dec, key, tm, rem, dft, xtra,
 							fmt, flg, dbf, v))
 							rc= HA_ERR_OUT_OF_MEM;
 				} // endfor i
@@ -7345,6 +7385,8 @@ static struct st_mysql_sys_var* connect_system_variables[]= {
   MYSQL_SYSVAR(errmsg_dir_path),
 #endif   // XMSG
 	MYSQL_SYSVAR(json_null),
+	MYSQL_SYSVAR(json_all_path),
+	MYSQL_SYSVAR(default_depth),
 	MYSQL_SYSVAR(json_grp_size),
 #if defined(JAVA_SUPPORT)
 	MYSQL_SYSVAR(jvm_path),
@@ -7371,7 +7413,7 @@ maria_declare_plugin(connect)
   0x0107,                                       /* version number (1.07) */
   NULL,                                         /* status variables */
   connect_system_variables,                     /* system variables */
-  "1.07.0001",                                  /* string version */
+  "1.07.0002",                                  /* string version */
 	MariaDB_PLUGIN_MATURITY_STABLE                /* maturity */
 }
 maria_declare_plugin_end;

storage/connect/ha_connect.h

@@ -104,6 +104,8 @@ struct ha_field_option_struct
   uint opt;
   const char *dateformat;
   const char *fieldformat;
+	const char* jsonpath;
+	const char* xmlpath;
 	char *special;
 };
 

storage/connect/json.cpp

@@ -93,9 +93,8 @@ char *NextChr(PSZ s, char sep)
 PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma)
 {
 	int   i, pretty = (ptyp) ? *ptyp : 3;
-	bool  b = false, pty[3] = {true, true, true};
-  PJSON jsp = NULL;
-  STRG  src;
+	bool  b = false, pty[3] = {true,true,true};
+  PJSON jsp = NULL, jp = NULL;
 
 	if (trace(1))
 		htrc("ParseJson: s=%.10s len=%d\n", s, len);
@@ -106,27 +105,29 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma)
   } else if (comma)
     *comma = false;
 
-  src.str = s;
-  src.len = len;
-
 	// Trying to guess the pretty format
 	if (s[0] == '[' && (s[1] == '\n' || (s[1] == '\r' && s[2] == '\n')))
 		pty[0] = false;
 
 	try {
-		for (i = 0; i < len; i++)
+		jp = new(g) JSON();
+		jp->s = s;
+		jp->len = len;
+		jp->pty = pty;
+
+		for (i = 0; i < jp->len; i++)
 			switch (s[i]) {
 				case '[':
 					if (jsp)
-						goto tryit;
-					else if (!(jsp = ParseArray(g, ++i, src, pty)))
-						throw 1;
+						jsp = jp->ParseAsArray(g, i, pretty, ptyp);
+					else
+						jsp = jp->ParseArray(g, ++i);
 
 					break;
 				case '{':
 					if (jsp)
-						goto tryit;
-					else if (!(jsp = ParseObject(g, ++i, src, pty)))
+						jsp = jp->ParseAsArray(g, i, pretty, ptyp);
+					else if (!(jsp = jp->ParseObject(g, ++i)))
 						throw 2;
 
 					break;
@@ -157,8 +158,8 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma)
 
 				default:
 					if (jsp)
-						goto tryit;
-					else if (!(jsp = ParseValue(g, i, src, pty)))
+						jsp = jp->ParseAsArray(g, i, pretty, ptyp);
+					else if (!(jsp = jp->ParseValue(g, i)))
 						throw 4;
 
 					break;
@@ -187,10 +188,17 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma)
 	} // end catch
 
 	return jsp;
+} // end of ParseJson
 
-tryit:
+/***********************************************************************/
+/* Parse several items as being in an array.                           */
+/***********************************************************************/
+PJAR JSON::ParseAsArray(PGLOBAL g, int& i, int pretty, int *ptyp)
+{
 	if (pty[0] && (!pretty || pretty > 2)) {
-		if ((jsp = ParseArray(g, (i = 0), src, pty)) && ptyp && pretty == 3)
+		PJAR jsp;
+
+		if ((jsp = ParseArray(g, (i = 0))) && ptyp && pretty == 3)
 			*ptyp = (pty[0]) ? 0 : 3;
 
 		return jsp;
@@ -198,26 +206,23 @@ PJSON ParseJson(PGLOBAL g, char *s, int len, int *ptyp, bool *comma)
 		strcpy(g->Message, "More than one item in file");
 
 	return NULL;
-} // end of ParseJson
+} // end of ParseAsArray
 
 /***********************************************************************/
 /* Parse a JSON Array.                                                 */
 /***********************************************************************/
-PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty)
+PJAR JSON::ParseArray(PGLOBAL g, int& i)
 {
-  char  *s = src.str;
-  int    len = src.len;
 	int  level = 0;
 	bool b = (!i);
   PJAR jarp = new(g) JARRAY;
-  PJVAL  jvp = NULL;
 
   for (; i < len; i++)
     switch (s[i]) {
       case ',':
         if (level < 2) {
           sprintf(g->Message, "Unexpected ',' near %.*s",ARGS);
-          return NULL;
+          throw 1;
         } else
           level = 1;
 
@@ -225,7 +230,7 @@ PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty)
       case ']':
         if (level == 1) {
           sprintf(g->Message, "Unexpected ',]' near %.*s", ARGS);
-          return NULL;
+          throw 1;
         } // endif level
 
         jarp->InitArray(g);
@@ -240,11 +245,9 @@ PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty)
       default:
         if (level == 2) {
           sprintf(g->Message, "Unexpected value near %.*s", ARGS);
-          return NULL;
-        } else if ((jvp = ParseValue(g, i, src, pty)))
-          jarp->AddValue(g, jvp);
-        else
-          return NULL;
+          throw 1;
+        } else
+          jarp->AddValue(g, ParseValue(g, i));
 
         level = (b) ? 1 : 2;
         break;
@@ -256,18 +259,15 @@ PJAR ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty)
 		return jarp;
 	} // endif b
 
-  strcpy(g->Message, "Unexpected EOF in array");
-  return NULL;
+  throw ("Unexpected EOF in array");
 } // end of ParseArray
 
 /***********************************************************************/
 /* Parse a JSON Object.                                                */
 /***********************************************************************/
-PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty)
+PJOB JSON::ParseObject(PGLOBAL g, int& i)
 {
   PSZ   key;
-  char *s = src.str;
-  int   len = src.len;
   int   level = 0;
   PJOB  jobp = new(g) JOBJECT;
   PJPR  jpp = NULL;
@@ -276,42 +276,37 @@ PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty)
     switch (s[i]) {
       case '"':
         if (level < 2) {
-          if ((key = ParseString(g, ++i, src))) {
+					key = ParseString(g, ++i);
           jpp = jobp->AddPair(g, key);
           level = 1;
-          } else
-            return NULL;
-
         } else {
           sprintf(g->Message, "misplaced string near %.*s", ARGS);
-          return NULL;
+          throw 2;
         } // endif level
 
         break;
       case ':':
         if (level == 1) {
-          if (!(jpp->Val = ParseValue(g, ++i, src, pty)))
-            return NULL;
-
+					jpp->Val = ParseValue(g, ++i);
           level = 2;
         } else {
           sprintf(g->Message, "Unexpected ':' near %.*s", ARGS);
-          return NULL;
+          throw 2;
         } // endif level
 
         break;
       case ',':
         if (level < 2) {
           sprintf(g->Message, "Unexpected ',' near %.*s", ARGS);
-          return NULL;
+          throw 2;
         } else
-          level = 1;
+          level = 0;
 
         break;
       case '}':
-        if (level == 1) {
+        if (level < 2) {
           sprintf(g->Message, "Unexpected '}' near %.*s", ARGS);
-          return NULL;
+          throw 2;
           } // endif level
 
         return jobp;
@@ -324,20 +319,19 @@ PJOB ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty)
       default:
         sprintf(g->Message, "Unexpected character '%c' near %.*s",
                 s[i], ARGS);
-        return NULL;
+        throw 2;
     }; // endswitch s[i]
 
   strcpy(g->Message, "Unexpected EOF in Object");
-  return NULL;
+  throw 2;
 } // end of ParseObject
 
 /***********************************************************************/
 /* Parse a JSON Value.                                                 */
 /***********************************************************************/
-PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty)
+PJVAL JSON::ParseValue(PGLOBAL g, int& i)
 {
-  char *strval, *s = src.str;
-  int   n, len = src.len;
+  int   n;
   PJVAL jvp = new(g) JVALUE;
 
   for (; i < len; i++)
@@ -355,21 +349,13 @@ PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty)
  suite:
   switch (s[i]) {
     case '[':
-      if (!(jvp->Jsp = ParseArray(g, ++i, src, pty)))
-        return NULL;
-
+			jvp->Jsp = ParseArray(g, ++i);
       break;
     case '{':
-      if (!(jvp->Jsp = ParseObject(g, ++i, src, pty)))
-        return NULL;
-
+			jvp->Jsp = ParseObject(g, ++i);
       break;
     case '"':
-      if ((strval = ParseString(g, ++i, src)))
-        jvp->Value = AllocateValue(g, strval, TYPE_STRING);
-      else
-        return NULL;
-
+      jvp->Value = AllocateValue(g, ParseString(g, ++i), TYPE_STRING);
       break;
     case 't':
       if (!strncmp(s + i, "true", 4)) {
@@ -398,11 +384,9 @@ PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty)
       break;
     case '-':
     default:
-      if (s[i] == '-' || isdigit(s[i])) {
-        if (!(jvp->Value = ParseNumeric(g, i, src)))
-          goto err;
-
-      } else
+      if (s[i] == '-' || isdigit(s[i]))
+				jvp->Value = ParseNumeric(g, i);
+      else
         goto err;
 
     }; // endswitch s[i]
@@ -410,25 +394,21 @@ PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty)
   return jvp;
 
 err:
-  sprintf(g->Message, "Unexpected character '%c' near %.*s",
-          s[i], ARGS);
-  return NULL;
+  sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS);
+  throw 3;
 } // end of ParseValue
 
 /***********************************************************************/
 /*  Unescape and parse a JSON string.                                  */
 /***********************************************************************/
-char *ParseString(PGLOBAL g, int& i, STRG& src)
+char *JSON::ParseString(PGLOBAL g, int& i)
 {
-  char  *s = src.str;
   uchar *p;
-  int    n = 0, len = src.len;
+  int    n = 0;
 
   // Be sure of memory availability
-  if (len + 1 - i > (signed)((PPOOLHEADER)g->Sarea)->FreeBlk) {
-    strcpy(g->Message, "ParseString: Out of memory");
-    return NULL;
-    } // endif len
+  if (((size_t)len + 1 - i) > ((PPOOLHEADER)g->Sarea)->FreeBlk)
+    throw("ParseString: Out of memory");
 
   // The size to allocate is not known yet
   p = (uchar*)PlugSubAlloc(g, NULL, 0);
@@ -502,17 +482,16 @@ char *ParseString(PGLOBAL g, int& i, STRG& src)
       }; // endswitch s[i]
 
  err:
-  strcpy(g->Message, "Unexpected EOF in String");
-  return NULL;
+  throw("Unexpected EOF in String");
 } // end of ParseString
 
 /***********************************************************************/
 /* Parse a JSON numeric value.                                         */
 /***********************************************************************/
-PVAL ParseNumeric(PGLOBAL g, int& i, STRG& src)
+PVAL JSON::ParseNumeric(PGLOBAL g, int& i)
 {
-  char *s = src.str, buf[50];
-  int   n = 0, len = src.len;
+  char  buf[50];
+  int   n = 0;
   short nd = 0;
 	bool  has_dot = false;
 	bool  has_e = false;
@@ -575,14 +554,11 @@ PVAL ParseNumeric(PGLOBAL g, int& i, STRG& src)
 
     i--;  // Unstack  following character
     return valp;
-  } else {
-    strcpy(g->Message, "No digit found");
-    return NULL;
-  } // endif found_digit
+  } else
+    throw("No digit found");
 
  err:
-  strcpy(g->Message, "Unexpected EOF in number");
-  return NULL;
+  throw("Unexpected EOF in number");
 } // end of ParseNumeric
 
 /***********************************************************************/

storage/connect/json.h

 /**************** json H Declares Source Code File (.H) ****************/
 /*  Name: json.h   Version 1.2                                         */
 /*                                                                     */
-/*  (C) Copyright to the author Olivier BERTRAND          2014 - 2017  */
+/*  (C) Copyright to the author Olivier BERTRAND          2014 - 2020  */
 /*                                                                     */
 /*  This file contains the JSON classes declares.                      */
 /***********************************************************************/
+#include <mysql_com.h>
 #include "value.h"
 
 #if defined(_DEBUG)
@@ -44,15 +45,31 @@ typedef struct {
   int   len;
   } STRG, *PSG;
 
+// BSON size should be equal on Linux and Windows
+#define BMX 255
+typedef struct BSON* PBSON;
+
+/***********************************************************************/
+/*  Structure used to return binary json to Json UDF functions.        */
+/***********************************************************************/
+struct BSON {
+	char    Msg[BMX + 1];
+	char   *Filename;
+	PGLOBAL G;
+	int     Pretty;
+	ulong   Reslen;
+	my_bool Changed;
+	PJSON   Top;
+	PJSON   Jsp;
+	PBSON   Bsp;
+}; // end of struct BSON
+
+PBSON JbinAlloc(PGLOBAL g, UDF_ARGS* args, ulong len, PJSON jsp);
+
 char *NextChr(PSZ s, char sep);
 char *GetJsonNull(void);
 
-PJSON ParseJson(PGLOBAL g, char *s, int n, int *prty = NULL, bool *b = NULL);
-PJAR  ParseArray(PGLOBAL g, int& i, STRG& src, bool *pty);
-PJOB  ParseObject(PGLOBAL g, int& i, STRG& src, bool *pty);
-PJVAL ParseValue(PGLOBAL g, int& i, STRG& src, bool *pty);
-char *ParseString(PGLOBAL g, int& i, STRG& src);
-PVAL  ParseNumeric(PGLOBAL g, int& i, STRG& src);
+PJSON ParseJson(PGLOBAL g, char* s, int n, int* prty = NULL, bool* b = NULL);
 PSZ   Serialize(PGLOBAL g, PJSON jsp, char *fn, int pretty);
 bool  SerializeArray(JOUT *js, PJAR jarp, bool b);
 bool  SerializeObject(JOUT *js, PJOB jobp);
@@ -130,7 +147,7 @@ class JOUTPRT : public JOUTFILE {
 class JPAIR : public BLOCK {
   friend class JOBJECT;
 	friend class JSNX;
-	friend PJOB ParseObject(PGLOBAL, int&, STRG&, bool*);
+	friend class JSON;
   friend bool SerializeObject(JOUT *, PJOB);
  public:
   JPAIR(PCSZ key) : BLOCK() {Key = key; Val = NULL; Next = NULL;}
@@ -149,8 +166,9 @@ class JPAIR : public BLOCK {
 /* Class JSON. The base class for all other json classes.              */
 /***********************************************************************/
 class JSON : public BLOCK {
+	friend PJSON ParseJson(PGLOBAL, char*, int, int*, bool*);
  public:
-  JSON(void) {Size = 0;}
+	 JSON(void) : s(NULL), len(0), pty(NULL) {Size = 0;}
 
           int    size(void) {return Size;}
 	virtual int    GetSize(bool b) {return Size;}
@@ -187,14 +205,27 @@ class JSON : public BLOCK {
 	virtual bool   IsNull(void) {X return true;}
 
  protected:
+	PJAR  ParseArray(PGLOBAL g, int& i);
+	PJOB  ParseObject(PGLOBAL g, int& i);
+	PJVAL ParseValue(PGLOBAL g, int& i);
+	char *ParseString(PGLOBAL g, int& i);
+	PVAL  ParseNumeric(PGLOBAL g, int& i);
+	PJAR  ParseAsArray(PGLOBAL g, int& i, int pretty, int *ptyp);
+
+	// Members
 	int Size;
+
+	// Only used when parsing
+ private:
+	char *s;
+	int   len;
+	bool *pty;
 }; // end of class JSON
 
 /***********************************************************************/
 /* Class JOBJECT: contains a list of value pairs.                      */
 /***********************************************************************/
 class JOBJECT : public JSON {
-  friend PJOB ParseObject(PGLOBAL, int&, STRG&, bool*);
   friend bool SerializeObject(JOUT *, PJOB);
 	friend class JSNX;
  public:
@@ -260,8 +291,8 @@ class JVALUE : public JSON {
   friend class JARRAY;
 	friend class JSNX;
 	friend class JSONCOL;
-	friend PJVAL ParseValue(PGLOBAL, int&, STRG&, bool*);
-  friend bool  SerializeValue(JOUT *, PJVAL);
+  friend class JSON;
+	friend bool SerializeValue(JOUT*, PJVAL);
  public:
   JVALUE(void) : JSON() {Clear();}
 	JVALUE(PJSON jsp);

storage/connect/jsonudf.cpp

@@ -25,7 +25,7 @@
 #else
 #define PUSH_WARNING(M) htrc(M)
 #endif
-#define M 7
+#define M 9
 
 bool  IsNum(PSZ s);
 char *NextChr(PSZ s, char sep);
@@ -1076,29 +1076,10 @@ my_bool JSNX::AddPath(void)
 
 /* --------------------------------- JSON UDF ---------------------------------- */
 
-// BSON size should be equal on Linux and Windows
-#define BMX 255
-typedef struct BSON *PBSON;
-
-/*********************************************************************************/
-/*  Structure used to return binary json.                                        */
-/*********************************************************************************/
-struct BSON {
-	char    Msg[BMX + 1];
-	char   *Filename;
-	PGLOBAL G;
-	int     Pretty;
-	ulong   Reslen;
-	my_bool Changed;
-	PJSON   Top;
-	PJSON   Jsp;
-	PBSON   Bsp;
-}; // end of struct BSON
-
 /*********************************************************************************/
 /*  Allocate and initialize a BSON structure.                                    */
 /*********************************************************************************/
-static PBSON JbinAlloc(PGLOBAL g, UDF_ARGS *args, ulong len, PJSON jsp)
+PBSON JbinAlloc(PGLOBAL g, UDF_ARGS *args, ulong len, PJSON jsp)
 {
 	PBSON bsp = (PBSON)PlgDBSubAlloc(g, NULL, sizeof(BSON));
 
@@ -1111,7 +1092,7 @@ static PBSON JbinAlloc(PGLOBAL g, UDF_ARGS *args, ulong len, PJSON jsp)
 		bsp->Reslen = len;
 		bsp->Changed = false;
 		bsp->Top = bsp->Jsp = jsp;
-		bsp->Bsp = (IsJson(args, 0) == 3) ? (PBSON)args->args[0] : NULL;
+		bsp->Bsp = (args && IsJson(args, 0) == 3) ? (PBSON)args->args[0] : NULL;
 	} else
 		PUSH_WARNING(g->Message);
 
@@ -1144,7 +1125,7 @@ static my_bool JsonSubSet(PGLOBAL g)
 {
 	PPOOLHEADER pph = (PPOOLHEADER)g->Sarea;
 
-	pph->To_Free = (OFFSET)((g->Createas) ? g->Createas : sizeof(POOLHEADER));
+	pph->To_Free = (g->Saved_Size) ? g->Saved_Size : (size_t)sizeof(POOLHEADER);
 	pph->FreeBlk = g->Sarea_Size - pph->To_Free;
 	return FALSE;
 } /* end of JsonSubSet */
@@ -1154,7 +1135,7 @@ static my_bool JsonSubSet(PGLOBAL g)
 /*********************************************************************************/
 inline void JsonMemSave(PGLOBAL g)
 {
-	g->Createas = (int)((PPOOLHEADER)g->Sarea)->To_Free;
+	g->Saved_Size = ((PPOOLHEADER)g->Sarea)->To_Free;
 } /* end of JsonMemSave */
 
 /*********************************************************************************/
@@ -1422,7 +1403,7 @@ static int IsJson(UDF_ARGS *args, uint i, bool b)
 		n = 2;					   //	arg is a json file name
 	} else if (b) {
 		char   *sap;
-		PGLOBAL g = PlugInit(NULL, args->lengths[i] * M + 1024);
+		PGLOBAL g = PlugInit(NULL, (size_t)args->lengths[i] * M + 1024);
 
 		JsonSubSet(g);
 		sap = MakePSZ(g, args, i);
@@ -1625,7 +1606,7 @@ static my_bool CheckMemory(PGLOBAL g, UDF_INIT *initid, UDF_ARGS *args, uint n,
 					return true;
 					} // endif SareaAlloc
 
-				g->Createas = 0;
+				g->Saved_Size = 0;
 				g->Xchk = NULL;
 				initid->max_length = rl;
 			}	// endif Size
@@ -4423,13 +4404,15 @@ char *json_file(UDF_INIT *initid, UDF_ARGS *args, char *result,
 	fn = MakePSZ(g, args, 0);
 
 	if (args->arg_count > 1) {
-		int    len, pretty, pty = 3;
+		int    len, pretty = 3, pty = 3;
 		PJSON  jsp;
 		PJVAL  jvp = NULL;
 
-		pretty = (args->arg_type[1] == INT_RESULT) ? (int)*(longlong*)args->args[1]
-					 : (args->arg_count > 2 && args->arg_type[2] == INT_RESULT)
-					 ? (int)*(longlong*)args->args[2] : 3;
+		for (unsigned int i = 1; i < args->arg_count; i++)
+			if (args->arg_type[i] == INT_RESULT && *(longlong*)args->args[i] < 4) {
+				pretty = (int) * (longlong*)args->args[i];
+				break;
+			} // endif type
 
 		/*******************************************************************************/
 		/*  Parse the json file and allocate its tree structure.                       */
@@ -4497,6 +4480,7 @@ my_bool jfile_make_init(UDF_INIT *initid, UDF_ARGS *args, char *message)
 	}	// endif
 
 	CalcLen(args, false, reslen, memlen);
+	memlen = memlen + 5000;	 // To take care of not pretty files 
 	return JsonInit(initid, args, message, true, reslen, memlen);
 } // end of jfile_make_init
 
@@ -5626,20 +5610,19 @@ my_bool jbin_file_init(UDF_INIT *initid, UDF_ARGS *args, char *message)
 	} else if (args->arg_type[0] != STRING_RESULT || !args->args[0]) {
 		strcpy(message, "First argument must be a constant string (file name)");
 		return true;
-	} else if (args->arg_count > 1 &&	args->arg_type[1] != STRING_RESULT) {
-		strcpy(message, "Second argument is not a string (path)");
-		return true;
-	} else if (args->arg_count > 2 &&	args->arg_type[2] != INT_RESULT) {
-		strcpy(message, "Third argument is not an integer (pretty)");
-		return true;
-	} else if (args->arg_count > 3) {
-		if (args->arg_type[3] != INT_RESULT) {
-			strcpy(message, "Fourth argument is not an integer (memory)");
+	} // endifs
+
+	for (unsigned int i = 1; i < args->arg_count; i++) {
+		if (!(args->arg_type[i] == INT_RESULT || args->arg_type[i] == STRING_RESULT)) {
+			sprintf(message, "Argument %d is not an integer or a string (pretty or path)", i);
 			return true;
-		} else
-			more += (ulong)*(longlong*)args->args[3];
+		} // endif arg_type
 
-	} // endifs
+		// Take care of eventual memory argument
+		if (args->arg_type[i] == INT_RESULT && args->args[i])
+			more += (ulong) * (longlong*)args->args[i];
+
+	} // endfor i
 
 	initid->maybe_null = 1;
 	CalcLen(args, false, reslen, memlen);
@@ -5654,7 +5637,7 @@ char *jbin_file(UDF_INIT *initid, UDF_ARGS *args, char *result,
 	unsigned long *res_length, char *is_null, char *error)
 {
 	char   *fn;
-	int     pretty, len = 0, pty = 3;
+	int     pretty = 3, len = 0, pty = 3;
 	PJSON   jsp;
 	PJVAL   jvp = NULL;
 	PGLOBAL g = (PGLOBAL)initid->ptr;
@@ -5666,7 +5649,12 @@ char *jbin_file(UDF_INIT *initid, UDF_ARGS *args, char *result,
 	PlugSubSet(g->Sarea, g->Sarea_Size);
 	g->Xchk = NULL;
 	fn = MakePSZ(g, args, 0);
-	pretty = (args->arg_count > 2 && args->args[2]) ? (int)*(longlong*)args->args[2] : 3;
+
+	for (unsigned int i = 1; i < args->arg_count; i++)
+		if (args->arg_type[i] == INT_RESULT && *(longlong*)args->args[i] < 4) {
+			pretty = (int) * (longlong*)args->args[i];
+			break;
+		} // endif type
 
 	/*********************************************************************************/
 	/*  Parse the json file and allocate its tree structure.                         */
@@ -5757,7 +5745,7 @@ char *json_serialize(UDF_INIT *initid, UDF_ARGS *args, char *result,
 			// Keep result of constant function
 			g->Xchk = (initid->const_item) ? str : NULL;
 		} else {
-			*error = 1;
+			// *error = 1;
 			str = strcpy(result, "Argument is not a Jbin tree");
 		} // endif
 
@@ -5773,6 +5761,478 @@ void json_serialize_deinit(UDF_INIT* initid)
 	JsonFreeMem((PGLOBAL)initid->ptr);
 } // end of json_serialize_deinit
 
+/*********************************************************************************/
+/*  Convert a prettiest Json file to Pretty=0.                                   */
+/*********************************************************************************/
+my_bool jfile_convert_init(UDF_INIT* initid, UDF_ARGS* args, char* message) {
+	unsigned long reslen, memlen;
+
+	if (args->arg_count != 3) {
+		strcpy(message, "This function must have 3 arguments");
+		return true;
+	} else if (args->arg_type[2] != INT_RESULT) {
+		strcpy(message, "Third Argument must be an integer (LRECL)");
+		return true;
+	} else for (int i = 0; i < 2; i++)
+		if (args->arg_type[i] != STRING_RESULT) {
+			sprintf(message, "Arguments %d must be a string (file name)", i+1);
+			return true;
+		} // endif args
+
+	CalcLen(args, false, reslen, memlen);
+	return JsonInit(initid, args, message, false, reslen, memlen);
+} // end of jfile_convert_init
+
+char *jfile_convert(UDF_INIT* initid, UDF_ARGS* args, char* result,
+	                  unsigned long *res_length, char *, char *error) {
+	char   *str, *fn, *ofn;
+	int     lrecl = (int)*(longlong*)args->args[2];
+	PGLOBAL g = (PGLOBAL)initid->ptr;
+
+	PlugSubSet(g->Sarea, g->Sarea_Size);
+	fn = MakePSZ(g, args, 0);
+	ofn = MakePSZ(g, args, 1);
+
+	if (!g->Xchk) {
+		JUP* jup = new(g) JUP(g);
+
+		str = jup->UnprettyJsonFile(g, fn, ofn, lrecl);
+		g->Xchk = str;
+	} else
+		str = (char*)g->Xchk;
+
+	if (!str) {
+		if (g->Message)
+			str = PlugDup(g, g->Message);
+		else
+			str = PlugDup(g, "Unexpected error");
+
+	} // endif str
+
+	*res_length = strlen(str);
+	return str;
+} // end of jfile_convert
+
+void jfile_convert_deinit(UDF_INIT* initid) {
+	JsonFreeMem((PGLOBAL)initid->ptr);
+} // end of jfile_convert_deinit
+
+/* --------------------------------- Class JUP --------------------------------- */
+
+#define ARGS       MY_MIN(24,len-i),s+MY_MAX(i-3,0)
+
+/*********************************************************************************/
+/*  JUP public constructor.                                                      */
+/*********************************************************************************/
+JUP::JUP(PGLOBAL g) {
+	fs = NULL;
+	s = buff = NULL;
+	i = k = len = recl = 0;
+} // end of JUP constructor
+
+/*********************************************************************************/
+/*  Copy a json file to another with pretty = 0.                                 */
+/*********************************************************************************/
+char* JUP::UnprettyJsonFile(PGLOBAL g, char *fn, char *outfn, int lrecl) {
+	char   *ret = NULL;
+	HANDLE  hFile;
+	MEMMAP  mm;
+
+	/*******************************************************************************/
+	/*  Create the mapping file object.                                            */
+	/*******************************************************************************/
+	hFile = CreateFileMap(g, fn, &mm, MODE_READ, false);
+
+	if (hFile == INVALID_HANDLE_VALUE) {
+		DWORD rc = GetLastError();
+
+		if (!(*g->Message))
+			sprintf(g->Message, MSG(OPEN_MODE_ERROR), "map", (int)rc, fn);
+
+		return NULL;
+	} // endif hFile
+
+	/*******************************************************************************/
+	/*  Get the file size (assuming file is smaller than 4 GB)                     */
+	/*******************************************************************************/
+	if (!mm.lenL) {              // Empty or deleted file
+		CloseFileHandle(hFile);
+		return NULL;
+	} else
+		len = (int)mm.lenL;
+
+	if (!mm.memory) {
+		CloseFileHandle(hFile);
+		sprintf(g->Message, MSG(MAP_VIEW_ERROR), fn, GetLastError());
+		return NULL;
+	} else
+		s = (char*)mm.memory;
+
+	CloseFileHandle(hFile);                    // Not used anymore
+
+	/*********************************************************************************/
+	/*  Parse the json file and allocate its tree structure.                         */
+	/*********************************************************************************/
+	if (!(fs = fopen(outfn, "wb"))) {
+		sprintf(g->Message, MSG(OPEN_MODE_ERROR),
+			"w", (int)errno, outfn);
+		strcat(strcat(g->Message, ": "), strerror(errno));
+		CloseMemMap(mm.memory, (size_t)mm.lenL);
+		return NULL;
+	} // endif fs
+
+	g->Message[0] = 0;
+
+	if (!unPretty(g, lrecl))
+		ret = outfn;
+
+	CloseMemMap(mm.memory, (size_t)mm.lenL);
+	fclose(fs);
+	return ret;
+} // end of UnprettyJsonFile
+
+/***********************************************************************/
+/* Translate a json file to pretty = 0.                                */
+/***********************************************************************/
+bool JUP::unPretty(PGLOBAL g, int lrecl) {
+	bool  go, next, rc = false;
+
+	if (trace(1))
+		htrc("UnPretty: s=%.10s len=%zd lrecl=%d\n", s, len, lrecl);
+
+	if (!s || !len) {
+		strcpy(g->Message, "Void JSON file");
+		return true;
+	} else if (*s != '[') {
+		// strcpy(g->Message, "JSON file is not an array");
+		s = strchr(s, '[');
+		// return true;
+	}	// endif s
+
+	i = 1;
+	go = next = true;
+
+	try {
+		// Allocate the record
+		buff = (char*)PlugSubAlloc(g, NULL, (size_t)lrecl + 3);
+		recl = lrecl;
+
+		do {
+			for (k = 0; go && i < len; i++)
+				switch (s[i]) {
+				case '{':
+					buff[k++] = s[i++];
+					CopyObject(g);
+					break;
+				case '[':
+					throw "JSON file is not an array of objects";
+					break;
+				case ' ':
+				case '\t':
+				case '\n':
+				case '\r':
+					break;
+				case ',':
+					go = false;
+					break;
+				case ']':
+					go = next = false;
+					break;
+				default:
+					sprintf(g->Message, "Unexpected '%c' near %.*s", s[i], ARGS);
+					throw 4;
+					break;
+				}; // endswitch s[i]
+
+			// Write the record
+#ifdef __win_
+			buff[k++] = '\r';
+#endif
+			buff[k++] = '\n';
+			buff[k] = 0;
+
+			if ((fputs(buff, fs)) == EOF) {
+				sprintf(g->Message, MSG(FPUTS_ERROR), strerror(errno));
+				throw 5;
+			} // endif EOF
+
+			go = true;
+		} while (next);
+
+	} catch (int n) {
+		if (trace(1))
+			htrc("Exception %d: %s\n", n, g->Message);
+		rc = true;
+	} catch (const char* msg) {
+		strcpy(g->Message, msg);
+		rc = true;
+	} // end catch
+
+	return rc;
+} // end of unPretty
+
+/***********************************************************************/
+/* Copy a JSON Object.                                                 */
+/***********************************************************************/
+void JUP::CopyObject(PGLOBAL g) {
+	int level = 0;
+
+	for (; i < len; i++)
+		switch (s[i]) {
+		case '"':
+			AddBuff(s[i++]);
+
+			if (level < 2) {
+				CopyString(g);
+				level = 1;
+			} else {
+				sprintf(g->Message, "misplaced string near %.*s", ARGS);
+				throw 3;
+			} // endif level
+
+			break;
+		case ':':
+			AddBuff(s[i++]);
+
+			if (level == 1) {
+				CopyValue(g);
+				level = 2;
+			} else {
+				sprintf(g->Message, "Unexpected ':' near %.*s", ARGS);
+				throw 3;
+			} // endif level
+
+			break;
+		case ',':
+			AddBuff(s[i]);
+
+			if (level < 2) {
+				sprintf(g->Message, "Unexpected ',' near %.*s", ARGS);
+				throw 3;
+			} else
+				level = 0;
+
+			break;
+		case '}':
+			AddBuff(s[i]);
+
+			if (level == 1) {
+				sprintf(g->Message, "Unexpected '}' near %.*s", ARGS);
+				throw 3;
+			} // endif level
+
+			return;
+		case '\n':
+		case '\r':
+		case ' ':
+		case '\t':
+			break;
+		default:
+			sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS);
+			throw 3;
+		}; // endswitch s[i]
+
+	throw "Unexpected EOF in Object";
+} // end of CopyObject
+
+/***********************************************************************/
+/* Copy a JSON Array.                                                  */
+/***********************************************************************/
+void JUP::CopyArray(PGLOBAL g) {
+	int level = 0;
+
+	for (; i < len; i++)
+		switch (s[i]) {
+		case ',':
+			if (level < 2) {
+				sprintf(g->Message, "Unexpected ',' near %.*s", ARGS);
+				throw 2;
+			} else
+				level = 1;
+
+			AddBuff(s[i]);
+			break;
+		case ']':
+			if (level == 1) {
+				sprintf(g->Message, "Unexpected ',]' near %.*s", ARGS);
+				throw 2;
+			} // endif level
+
+			AddBuff(s[i]);
+			return;
+		case '\n':
+		case '\r':
+		case ' ':
+		case '\t':
+			break;
+		default:
+			if (level == 2) {
+				sprintf(g->Message, "Unexpected value near %.*s", ARGS);
+				throw 2;
+			} // endif level
+
+			CopyValue(g);
+			level = 2;
+			break;
+		}; // endswitch s[i]
+
+	throw "Unexpected EOF in array";
+} // end of CopyArray
+
+/***********************************************************************/
+/* Copy a JSON Value.                                                  */
+/***********************************************************************/
+void JUP::CopyValue(PGLOBAL g) {
+	for (; i < len; i++)
+		switch (s[i]) {
+		case '\n':
+		case '\r':
+		case ' ':
+		case '\t':
+			break;
+		default:
+			goto suite;
+		} // endswitch
+
+suite:
+	switch (s[i]) {
+	case '[':
+		AddBuff(s[i++]);
+		CopyArray(g);
+		break;
+	case '{':
+		AddBuff(s[i++]);
+		CopyObject(g);
+		break;
+	case '"':
+		AddBuff(s[i++]);
+		CopyString(g);
+		break;
+	case 't':
+		if (!strncmp(s + i, "true", 4)) {
+			AddBuff(s[i++]);
+			AddBuff(s[i++]);
+			AddBuff(s[i++]);
+			AddBuff(s[i]);
+		} else
+			goto err;
+
+		break;
+	case 'f':
+		if (!strncmp(s + i, "false", 5)) {
+			AddBuff(s[i++]);
+			AddBuff(s[i++]);
+			AddBuff(s[i++]);
+			AddBuff(s[i++]);
+			AddBuff(s[i]);
+		} else
+			goto err;
+
+		break;
+	case 'n':
+		if (!strncmp(s + i, "null", 4)) {
+			AddBuff(s[i++]);
+			AddBuff(s[i++]);
+			AddBuff(s[i++]);
+			AddBuff(s[i]);
+		} else
+			goto err;
+
+		break;
+	default:
+		if (s[i] == '-' || isdigit(s[i]))
+			CopyNumeric(g);
+		else
+			goto err;
+
+	}; // endswitch s[i]
+
+	return;
+
+err:
+	sprintf(g->Message, "Unexpected character '%c' near %.*s", s[i], ARGS);
+	throw 1;
+} // end of CopyValue
+
+/***********************************************************************/
+/*  Unescape and parse a JSON string.                                  */
+/***********************************************************************/
+void JUP::CopyString(PGLOBAL g) {
+	for (; i < len; i++) {
+		AddBuff(s[i]);
+
+		switch (s[i]) {
+		case '"':
+			return;
+		case '\\':
+			AddBuff(s[++i]);
+			break;
+		default:
+			break;
+		}; // endswitch s[i]
+
+	} // endfor i
+
+	throw "Unexpected EOF in String";
+} // end of CopyString
+
+/***********************************************************************/
+/* Copy a JSON numeric value.                                          */
+/***********************************************************************/
+void JUP::CopyNumeric(PGLOBAL g) {
+	bool  has_dot = false;
+	bool  has_e = false;
+	bool  found_digit = false;
+
+	for (; i < len; i++) {
+		switch (s[i]) {
+		case '.':
+			if (!found_digit || has_dot || has_e)
+				goto err;
+
+			has_dot = true;
+			break;
+		case 'e':
+		case 'E':
+			if (!found_digit || has_e)
+				goto err;
+
+			has_e = true;
+			found_digit = false;
+			break;
+		case '+':
+			if (!has_e)
+				goto err;
+
+			// fall through
+		case '-':
+			if (found_digit)
+				goto err;
+
+			break;
+		default:
+			if (isdigit(s[i])) {
+				found_digit = true;
+			} else
+				goto fin;
+
+		}; // endswitch s[i]
+
+		AddBuff(s[i]);
+	} // endfor i
+
+fin:
+	if (!found_digit)
+		throw "No digit found";
+	else
+		i--;
+
+	return;
+
+err:
+	throw "Unexpected EOF in number";
+} // end of CopyNumeric
+
 /*********************************************************************************/
 /*  Utility function returning an environment variable value.                    */
 /*********************************************************************************/

storage/connect/jsonudf.h

@@ -235,6 +235,10 @@ extern "C" {
 	DllExport char *json_serialize(UDF_EXEC_ARGS);
 	DllExport void json_serialize_deinit(UDF_INIT*);
 
+	DllExport my_bool jfile_convert_init(UDF_INIT*, UDF_ARGS*, char*);
+	DllExport char* jfile_convert(UDF_EXEC_ARGS);
+	DllExport void jfile_convert_deinit(UDF_INIT*);
+
 	DllExport my_bool envar_init(UDF_INIT*, UDF_ARGS*, char*);
 	DllExport char *envar(UDF_EXEC_ARGS);
 
@@ -324,3 +328,38 @@ class JSNX : public BLOCK {
 	my_bool  Wr;			  					// Write mode
 	my_bool  Jb;			  					// Must return json item
 }; // end of class JSNX
+
+/*********************************************************************************/
+/*  Class JUP: used by jfile_convert to make a json file pretty = 0.             */
+/*********************************************************************************/
+class JUP : public BLOCK {
+public:
+	// Constructor
+	JUP(PGLOBAL g);
+
+	// Implementation
+	void  AddBuff(char c) {
+		if (k < recl)
+			buff[k++] = c;
+		else
+			throw "Record size is too small";
+	}	// end of AddBuff
+
+	// Methods
+	char *UnprettyJsonFile(PGLOBAL g, char* fn, char* outfn, int lrecl);
+	bool  unPretty(PGLOBAL g, int lrecl);
+	void  CopyObject(PGLOBAL g);
+	void  CopyArray(PGLOBAL g);
+	void  CopyValue(PGLOBAL g);
+	void  CopyString(PGLOBAL g);
+	void  CopyNumeric(PGLOBAL g);
+
+	// Members
+	FILE* fs;
+	char* s;
+	char* buff;
+	int len;
+	int recl;
+	int i, k;
+}; // end of class JUP
+

storage/connect/mongo.cpp

@@ -35,6 +35,7 @@
 
 bool MakeSelector(PGLOBAL g, PFIL fp, PSTRG s);
 bool IsNum(PSZ s);
+int  GetDefaultDepth(void);
 
 /***********************************************************************/
 /*  Make selector json representation for Mongo tables.                */
@@ -248,15 +249,10 @@ MGODISC::MGODISC(PGLOBAL g, int *lg) {
 /***********************************************************************/
 int MGODISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ uri, PTOS topt)
 {
-	PCSZ    level = GetStringTableOption(g, topt, "Level", NULL);
 	PMGODEF tdp;
 
-	if (level) {
-		lvl = atoi(level);
-		lvl = (lvl > 16) ? 16 : lvl;
-	} else
-		lvl = 0;
-
+	lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth());
+	lvl = GetIntegerTableOption(g, topt, "Depth", lvl);
 	all = GetBooleanTableOption(g, topt, "Fullarray", false);
 
 	/*********************************************************************/

storage/connect/mysql-test/connect/r/json_java_2.result

@@ -20,12 +20,12 @@ SELECT * from t1;
 Column_Name	Data_Type	Type_Name	Column_Size	Buffer_Length	Decimal_Digits	Nullable	Jpath
 _id	1	CHAR	24	24	0	0	_id
 address_building	1	CHAR	10	10	0	0	address.building
-address_coord	1	CHAR	256	256	0	1	address.coord
+address_coord	1	CHAR	1024	1024	0	1	address.coord
 address_street	1	CHAR	38	38	0	0	address.street
 address_zipcode	1	CHAR	5	5	0	0	address.zipcode
 borough	1	CHAR	13	13	0	0	
 cuisine	1	CHAR	64	64	0	0	
-grades_date	1	CHAR	256	256	0	1	grades.0.date
+grades_date	1	CHAR	1024	1024	0	1	grades.0.date
 grades_grade	1	CHAR	14	14	0	1	grades.0.grade
 grades_score	5	BIGINT	2	2	0	1	grades.0.score
 name	1	CHAR	98	98	0	0	
@@ -64,16 +64,16 @@ OPTION_LIST='Level=1,Driver=Java,Version=2' CONNECTION='mongodb://localhost:2701
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `_id` char(24) NOT NULL `FIELD_FORMAT`='_id',
-  `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building',
-  `address_coord` varchar(256) DEFAULT NULL `FIELD_FORMAT`='address.coord',
-  `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street',
-  `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode',
+  `_id` char(24) NOT NULL `JPATH`='_id',
+  `address_building` char(10) NOT NULL `JPATH`='address.building',
+  `address_coord` varchar(1024) DEFAULT NULL `JPATH`='address.coord',
+  `address_street` char(38) NOT NULL `JPATH`='address.street',
+  `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode',
   `borough` char(13) NOT NULL,
   `cuisine` char(64) NOT NULL,
-  `grades_date` varchar(256) DEFAULT NULL `FIELD_FORMAT`='grades.0.date',
-  `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade',
-  `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score',
+  `grades_date` varchar(1024) DEFAULT NULL `JPATH`='grades.0.date',
+  `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade',
+  `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score',
   `name` char(98) NOT NULL,
   `restaurant_id` char(8) NOT NULL
 ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `OPTION_LIST`='Level=1,Driver=Java,Version=2' `DATA_CHARSET`='utf8' `LRECL`=4096
@@ -251,15 +251,15 @@ OPTION_LIST='Driver=Java,level=2,version=2';
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `_id` char(24) NOT NULL `FIELD_FORMAT`='_id',
-  `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building',
-  `address_coord` double(18,16) DEFAULT NULL `FIELD_FORMAT`='address.coord.0',
-  `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street',
-  `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode',
+  `_id` char(24) NOT NULL `JPATH`='_id',
+  `address_building` char(10) NOT NULL `JPATH`='address.building',
+  `address_coord` double(18,16) DEFAULT NULL `JPATH`='address.coord.0',
+  `address_street` char(38) NOT NULL `JPATH`='address.street',
+  `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode',
   `borough` char(13) NOT NULL,
-  `grades_date` char(24) DEFAULT NULL `FIELD_FORMAT`='grades.0.date',
-  `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade',
-  `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score',
+  `grades_date` char(24) DEFAULT NULL `JPATH`='grades.0.date',
+  `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade',
+  `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score',
   `name` char(98) NOT NULL,
   `restaurant_id` char(8) NOT NULL
 ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `COLIST`='{"cuisine":0}' `FILTER`='{"cuisine":"French","borough":{"$ne":"Manhattan"}}' `OPTION_LIST`='Driver=Java,level=2,version=2' `LRECL`=4096

storage/connect/mysql-test/connect/r/json_java_3.result

@@ -20,12 +20,12 @@ SELECT * from t1;
 Column_Name	Data_Type	Type_Name	Column_Size	Buffer_Length	Decimal_Digits	Nullable	Jpath
 _id	1	CHAR	24	24	0	0	_id
 address_building	1	CHAR	10	10	0	0	address.building
-address_coord	1	CHAR	256	256	0	1	address.coord
+address_coord	1	CHAR	1024	1024	0	1	address.coord
 address_street	1	CHAR	38	38	0	0	address.street
 address_zipcode	1	CHAR	5	5	0	0	address.zipcode
 borough	1	CHAR	13	13	0	0	
 cuisine	1	CHAR	64	64	0	0	
-grades_date	1	CHAR	256	256	0	1	grades.0.date
+grades_date	1	CHAR	1024	1024	0	1	grades.0.date
 grades_grade	1	CHAR	14	14	0	1	grades.0.grade
 grades_score	5	BIGINT	2	2	0	1	grades.0.score
 name	1	CHAR	98	98	0	0	
@@ -64,16 +64,16 @@ OPTION_LIST='Level=1,Driver=Java,Version=3' CONNECTION='mongodb://localhost:2701
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `_id` char(24) NOT NULL `FIELD_FORMAT`='_id',
-  `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building',
-  `address_coord` varchar(256) DEFAULT NULL `FIELD_FORMAT`='address.coord',
-  `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street',
-  `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode',
+  `_id` char(24) NOT NULL `JPATH`='_id',
+  `address_building` char(10) NOT NULL `JPATH`='address.building',
+  `address_coord` varchar(1024) DEFAULT NULL `JPATH`='address.coord',
+  `address_street` char(38) NOT NULL `JPATH`='address.street',
+  `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode',
   `borough` char(13) NOT NULL,
   `cuisine` char(64) NOT NULL,
-  `grades_date` varchar(256) DEFAULT NULL `FIELD_FORMAT`='grades.0.date',
-  `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade',
-  `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score',
+  `grades_date` varchar(1024) DEFAULT NULL `JPATH`='grades.0.date',
+  `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade',
+  `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score',
   `name` char(98) NOT NULL,
   `restaurant_id` char(8) NOT NULL
 ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `OPTION_LIST`='Level=1,Driver=Java,Version=3' `DATA_CHARSET`='utf8' `LRECL`=4096
@@ -251,15 +251,15 @@ OPTION_LIST='Driver=Java,level=2,version=3';
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `_id` char(24) NOT NULL `FIELD_FORMAT`='_id',
-  `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building',
-  `address_coord` double(18,16) DEFAULT NULL `FIELD_FORMAT`='address.coord.0',
-  `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street',
-  `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode',
+  `_id` char(24) NOT NULL `JPATH`='_id',
+  `address_building` char(10) NOT NULL `JPATH`='address.building',
+  `address_coord` double(18,16) DEFAULT NULL `JPATH`='address.coord.0',
+  `address_street` char(38) NOT NULL `JPATH`='address.street',
+  `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode',
   `borough` char(13) NOT NULL,
-  `grades_date` bigint(13) DEFAULT NULL `FIELD_FORMAT`='grades.0.date',
-  `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade',
-  `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score',
+  `grades_date` bigint(13) DEFAULT NULL `JPATH`='grades.0.date',
+  `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade',
+  `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score',
   `name` char(98) NOT NULL,
   `restaurant_id` char(8) NOT NULL
 ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `COLIST`='{"cuisine":0}' `FILTER`='{"cuisine":"French","borough":{"$ne":"Manhattan"}}' `OPTION_LIST`='Driver=Java,level=2,version=3' `LRECL`=4096

storage/connect/mysql-test/connect/r/json_mongo_c.result

@@ -20,12 +20,12 @@ SELECT * from t1;
 Column_Name	Data_Type	Type_Name	Column_Size	Buffer_Length	Decimal_Digits	Nullable	Jpath
 _id	1	CHAR	24	24	0	0	_id
 address_building	1	CHAR	10	10	0	0	address.building
-address_coord	1	CHAR	256	256	0	1	address.coord
+address_coord	1	CHAR	1024	1024	0	1	address.coord
 address_street	1	CHAR	38	38	0	0	address.street
 address_zipcode	1	CHAR	5	5	0	0	address.zipcode
 borough	1	CHAR	13	13	0	0	
 cuisine	1	CHAR	64	64	0	0	
-grades_date	1	CHAR	256	256	0	1	grades.0.date
+grades_date	1	CHAR	1024	1024	0	1	grades.0.date
 grades_grade	1	CHAR	14	14	0	1	grades.0.grade
 grades_score	5	BIGINT	2	2	0	1	grades.0.score
 name	1	CHAR	98	98	0	0	
@@ -64,16 +64,16 @@ OPTION_LIST='Level=1,Driver=C,Version=0' CONNECTION='mongodb://localhost:27017'
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `_id` char(24) NOT NULL `FIELD_FORMAT`='_id',
-  `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building',
-  `address_coord` varchar(256) DEFAULT NULL `FIELD_FORMAT`='address.coord',
-  `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street',
-  `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode',
+  `_id` char(24) NOT NULL `JPATH`='_id',
+  `address_building` char(10) NOT NULL `JPATH`='address.building',
+  `address_coord` varchar(1024) DEFAULT NULL `JPATH`='address.coord',
+  `address_street` char(38) NOT NULL `JPATH`='address.street',
+  `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode',
   `borough` char(13) NOT NULL,
   `cuisine` char(64) NOT NULL,
-  `grades_date` varchar(256) DEFAULT NULL `FIELD_FORMAT`='grades.0.date',
-  `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade',
-  `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score',
+  `grades_date` varchar(1024) DEFAULT NULL `JPATH`='grades.0.date',
+  `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade',
+  `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score',
   `name` char(98) NOT NULL,
   `restaurant_id` char(8) NOT NULL
 ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `OPTION_LIST`='Level=1,Driver=C,Version=0' `DATA_CHARSET`='utf8' `LRECL`=1024
@@ -251,15 +251,15 @@ OPTION_LIST='Driver=C,level=2,version=0';
 SHOW CREATE TABLE t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
-  `_id` char(24) NOT NULL `FIELD_FORMAT`='_id',
-  `address_building` char(10) NOT NULL `FIELD_FORMAT`='address.building',
-  `address_coord` double(23,20) DEFAULT NULL `FIELD_FORMAT`='address.coord.0',
-  `address_street` char(38) NOT NULL `FIELD_FORMAT`='address.street',
-  `address_zipcode` char(5) NOT NULL `FIELD_FORMAT`='address.zipcode',
+  `_id` char(24) NOT NULL `JPATH`='_id',
+  `address_building` char(10) NOT NULL `JPATH`='address.building',
+  `address_coord` double(23,20) DEFAULT NULL `JPATH`='address.coord.0',
+  `address_street` char(38) NOT NULL `JPATH`='address.street',
+  `address_zipcode` char(5) NOT NULL `JPATH`='address.zipcode',
   `borough` char(13) NOT NULL,
-  `grades_date` bigint(13) DEFAULT NULL `FIELD_FORMAT`='grades.0.date',
-  `grades_grade` char(14) DEFAULT NULL `FIELD_FORMAT`='grades.0.grade',
-  `grades_score` bigint(2) DEFAULT NULL `FIELD_FORMAT`='grades.0.score',
+  `grades_date` bigint(13) DEFAULT NULL `JPATH`='grades.0.date',
+  `grades_grade` char(14) DEFAULT NULL `JPATH`='grades.0.grade',
+  `grades_score` bigint(2) DEFAULT NULL `JPATH`='grades.0.score',
   `name` char(98) NOT NULL,
   `restaurant_id` char(8) NOT NULL
 ) ENGINE=CONNECT DEFAULT CHARSET=latin1 CONNECTION='mongodb://localhost:27017' `TABLE_TYPE`='JSON' `TABNAME`='restaurants' `COLIST`='{"projection":{"cuisine":0}}' `FILTER`='{"cuisine":"French","borough":{"$ne":"Manhattan"}}' `OPTION_LIST`='Driver=C,level=2,version=0' `LRECL`=1024

storage/connect/mysql-test/connect/r/updelx.result

@@ -978,7 +978,7 @@ DROP TABLE t1;
 # FIX table
 CREATE TABLE t1 (
 id INT(4) KEY NOT NULL,
-msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED)
+msg VARCHAR(16) DISTRIB=CLUSTERED)
 ENGINE=CONNECT TABLE_TYPE=FIX BLOCK_SIZE=4;
 Warnings:
 Warning	1105	No file name. Table will use t1.fix
@@ -1345,7 +1345,7 @@ DROP TABLE t1;
 # BIN table
 CREATE TABLE t1 (
 id INT(4) KEY NOT NULL,
-msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED)
+msg VARCHAR(16) DISTRIB=CLUSTERED)
 ENGINE=CONNECT TABLE_TYPE=BIN BLOCK_SIZE=8;
 Warnings:
 Warning	1105	No file name. Table will use t1.bin

storage/connect/mysql-test/connect/t/updelx.test

@@ -36,7 +36,7 @@ DROP TABLE t1;
 --echo # FIX table
 CREATE TABLE t1 (
 id INT(4) KEY NOT NULL,
-msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED)
+msg VARCHAR(16) DISTRIB=CLUSTERED)
 ENGINE=CONNECT TABLE_TYPE=FIX BLOCK_SIZE=4;
 -- source updelx.inc
 ALTER TABLE t1 MAPPED=YES;
@@ -48,7 +48,7 @@ DROP TABLE t1;
 --echo # BIN table
 CREATE TABLE t1 (
 id INT(4) KEY NOT NULL,
-msg VARCHAR(16) CHARSET BINARY DISTRIB=CLUSTERED)
+msg VARCHAR(16) DISTRIB=CLUSTERED)
 ENGINE=CONNECT TABLE_TYPE=BIN BLOCK_SIZE=8;
 -- source updelx.inc
 ALTER TABLE t1 MAPPED=YES;

storage/connect/myutil.h

@@ -6,8 +6,8 @@
 
 enum enum_field_types PLGtoMYSQL(int type, bool dbf, char var = 0);
 const char *PLGtoMYSQLtype(int type, bool dbf, char var = 0);
-int  MYSQLtoPLG(char *typname, char *var = NULL);
-int  MYSQLtoPLG(int mytype, char *var = NULL);
+int  MYSQLtoPLG(char *typname, char *var);
+int  MYSQLtoPLG(int mytype, char *var);
 PCSZ MyDateFmt(int mytype);
 PCSZ MyDateFmt(char *typname);
 

storage/connect/plgdbutl.cpp

@@ -5,7 +5,7 @@
 /*                                                                     */
 /* COPYRIGHT:                                                          */
 /* ----------                                                          */
-/*  (C) Copyright to the author Olivier BERTRAND          1998-2018    */
+/*  (C) Copyright to the author Olivier BERTRAND          1998-2020    */
 /*                                                                     */
 /* WHAT THIS PROGRAM DOES:                                             */
 /* -----------------------                                             */
@@ -1242,7 +1242,7 @@ void *PlgDBalloc(PGLOBAL g, void *area, MBLOCK& mp)
   mp.Sub = mp.Size <= ((mp.Sub) ? maxsub : (maxsub >> 2));
 
 	if (trace(2))
-		htrc("PlgDBalloc: in %p size=%d used=%d free=%d sub=%d\n",
+		htrc("PlgDBalloc: in %p size=%zd used=%zd free=%zd sub=%d\n",
 			arp, mp.Size, pph->To_Free, pph->FreeBlk, mp.Sub);
 
 	if (!mp.Sub) {
@@ -1258,7 +1258,7 @@ void *PlgDBalloc(PGLOBAL g, void *area, MBLOCK& mp)
       mp.Memp = malloc(mp.Size);
 
 		if (trace(8))
-			htrc("PlgDBalloc: %s(%d) at %p\n", v, mp.Size, mp.Memp);
+			htrc("PlgDBalloc: %s(%zd) at %p\n", v, mp.Size, mp.Memp);
 
 		if (!mp.Inlist && mp.Memp) {
       // New allocated block, put it in the memory block chain.
@@ -1290,7 +1290,7 @@ void *PlgDBrealloc(PGLOBAL g, void *area, MBLOCK& mp, size_t newsize)
 #endif
 
   if (trace(2))
-    htrc("PlgDBrealloc: %p size=%d sub=%d\n", mp.Memp, mp.Size, mp.Sub);
+    htrc("PlgDBrealloc: %p size=%zd sub=%d\n", mp.Memp, mp.Size, mp.Sub);
 
   if (newsize == mp.Size)
     return mp.Memp;      // Nothing to do
@@ -1340,7 +1340,7 @@ void *PlgDBrealloc(PGLOBAL g, void *area, MBLOCK& mp, size_t newsize)
   } // endif's
 
   if (trace(8))
-    htrc(" newsize=%d newp=%p sub=%d\n", mp.Size, mp.Memp, mp.Sub);
+    htrc(" newsize=%zd newp=%p sub=%d\n", mp.Size, mp.Memp, mp.Sub);
 
   return mp.Memp;
   } // end of PlgDBrealloc
@@ -1392,13 +1392,13 @@ void *PlgDBSubAlloc(PGLOBAL g, void *memp, size_t size)
   pph = (PPOOLHEADER)memp;
 
   if (trace(16))
-    htrc("PlgDBSubAlloc: memp=%p size=%d used=%d free=%d\n",
+    htrc("PlgDBSubAlloc: memp=%p size=%zd used=%zd free=%zd\n",
          memp, size, pph->To_Free, pph->FreeBlk);
 
-  if ((uint)size > pph->FreeBlk) {   /* Not enough memory left in pool */
+  if (size > pph->FreeBlk) {   /* Not enough memory left in pool */
     sprintf(g->Message,
-    "Not enough memory in Work area for request of %d (used=%d free=%d)",
-            (int) size, pph->To_Free, pph->FreeBlk);
+    "Not enough memory in Work area for request of %zd (used=%zd free=%zd)",
+            size, pph->To_Free, pph->FreeBlk);
 
     if (trace(1))
       htrc("%s\n", g->Message);
@@ -1414,7 +1414,7 @@ void *PlgDBSubAlloc(PGLOBAL g, void *memp, size_t size)
   pph->FreeBlk -= size;                 // New size   of pool free block
 
   if (trace(16))
-    htrc("Done memp=%p used=%d free=%d\n",
+    htrc("Done memp=%p used=%zd free=%zd\n",
          memp, pph->To_Free, pph->FreeBlk);
 
   return (memp);

storage/connect/plugutil.cpp

@@ -6,7 +6,7 @@
 /*                                                                     */
 /* COPYRIGHT:                                                          */
 /* ----------                                                          */
-/*  (C) Copyright to the author Olivier BERTRAND          1993-2019    */
+/*  (C) Copyright to the author Olivier BERTRAND          1993-2020    */
 /*                                                                     */
 /* WHAT THIS PROGRAM DOES:                                             */
 /* -----------------------                                             */
@@ -142,7 +142,7 @@ void htrc(char const* fmt, ...)
 /*  Language points on initial language name and eventual path.        */
 /*  Return value is the pointer to the Global structure.               */
 /***********************************************************************/
-PGLOBAL PlugInit(LPCSTR Language, uint worksize)
+PGLOBAL PlugInit(LPCSTR Language, size_t worksize)
 {
 	PGLOBAL g;
 
@@ -158,13 +158,14 @@ PGLOBAL PlugInit(LPCSTR Language, uint worksize)
 	} // end try/catch
 
 	g->Sarea = NULL;
-	g->Createas = 0;
+	g->Createas = false;
 	g->Alchecked = 0;
 	g->Mrr = 0;
 	g->Activityp = NULL;
 	g->Xchk = NULL;
 	g->N = 0;
 	g->More = 0;
+	g->Saved_Size = 0;
 	strcpy(g->Message, "");
 
 	/*******************************************************************/
@@ -459,7 +460,7 @@ short GetLineLength(PGLOBAL g)
 /***********************************************************************/
 /*  Program for memory allocation of work and language areas.          */
 /***********************************************************************/
-bool AllocSarea(PGLOBAL g, uint size)
+bool AllocSarea(PGLOBAL g, size_t size)
 {
   /*********************************************************************/
   /*  This is the allocation routine for the WIN32/UNIX/AIX version.   */
@@ -483,7 +484,7 @@ bool AllocSarea(PGLOBAL g, uint size)
 	if (trace(8)) {
 #endif
     if (g->Sarea)
-      htrc("Work area of %u allocated at %p\n", size, g->Sarea);
+      htrc("Work area of %zd allocated at %p\n", size, g->Sarea);
     else
       htrc("SareaAlloc: %s\n", g->Message);
 
@@ -510,7 +511,7 @@ void FreeSarea(PGLOBAL g)
 #else
 		if (trace(8))
 #endif
-			htrc("Freeing Sarea at %p size = %d\n", g->Sarea, g->Sarea_Size);
+			htrc("Freeing Sarea at %p size = %zd\n", g->Sarea, g->Sarea_Size);
 
 		g->Sarea = NULL;
 		g->Sarea_Size = 0;
@@ -524,11 +525,11 @@ void FreeSarea(PGLOBAL g)
 /*  Here there should be some verification done such as validity of    */
 /*  the address and size not larger than memory size.                  */
 /***********************************************************************/
-BOOL PlugSubSet(void *memp, uint size)
+BOOL PlugSubSet(void *memp, size_t size)
   {
   PPOOLHEADER pph = (PPOOLHEADER)memp;
 
-  pph->To_Free = (OFFSET)sizeof(POOLHEADER);
+  pph->To_Free = (size_t)sizeof(POOLHEADER);
   pph->FreeBlk = size - pph->To_Free;
   return FALSE;
   } /* end of PlugSubSet */
@@ -560,15 +561,15 @@ void *PlugSubAlloc(PGLOBAL g, void *memp, size_t size)
   pph = (PPOOLHEADER)memp;
 
   if (trace(16))
-    htrc("SubAlloc in %p size=%d used=%d free=%d\n",
+    htrc("SubAlloc in %p size=%zd used=%zd free=%zd\n",
           memp, size, pph->To_Free, pph->FreeBlk);
 
-  if ((uint)size > pph->FreeBlk) {   /* Not enough memory left in pool */
+  if (size > pph->FreeBlk) {   /* Not enough memory left in pool */
     PCSZ pname = "Work";
 
     sprintf(g->Message,
-      "Not enough memory in %s area for request of %u (used=%d free=%d)",
-                          pname, (uint)size, pph->To_Free, pph->FreeBlk);
+      "Not enough memory in %s area for request of %zd (used=%zd free=%zd)",
+                          pname, size, pph->To_Free, pph->FreeBlk);
 
     if (trace(1))
       htrc("PlugSubAlloc: %s\n", g->Message);
@@ -580,11 +581,11 @@ void *PlugSubAlloc(PGLOBAL g, void *memp, size_t size)
   /*  Do the suballocation the simplest way.                           */
   /*********************************************************************/
   memp = MakePtr(memp, pph->To_Free); /* Points to suballocated block  */
-  pph->To_Free += (OFFSET)size;       /* New offset of pool free block */
-  pph->FreeBlk -= (uint)size;         /* New size   of pool free block */
+  pph->To_Free += size;               /* New offset of pool free block */
+  pph->FreeBlk -= size;               /* New size   of pool free block */
 
   if (trace(16))
-    htrc("Done memp=%p used=%d free=%d\n",
+    htrc("Done memp=%p used=%zd free=%zd\n",
           memp, pph->To_Free, pph->FreeBlk);
 
   return (memp);
@@ -605,40 +606,4 @@ char *PlugDup(PGLOBAL g, const char *str)
 
   } // end of PlugDup 
 
-#if 0
-/***********************************************************************/
-/* This routine suballocate a copy of the passed string.               */
-/***********************************************************************/
-char *PlugDup(PGLOBAL g, const char *str)
-  {
-  char  *buf;
-  size_t len;
-
-  if (str && (len = strlen(str))) {
-    buf = (char*)PlugSubAlloc(g, NULL, len + 1);
-    strcpy(buf, str);
-  } else
-    buf = NULL;
-
-  return(buf);
-  } /* end of PlugDup */
-#endif // 0
-
-/***********************************************************************/
-/* This routine makes a pointer from an offset to a memory pointer.    */
-/***********************************************************************/
-void *MakePtr(void *memp, OFFSET offset)
-  {
-  return ((offset == 0) ? NULL : &((char *)memp)[offset]);
-  } /* end of MakePtr */
-
-/***********************************************************************/
-/* This routine makes an offset from a pointer new format.             */
-/***********************************************************************/
-#if 0
-OFFSET MakeOff(void *memp, void *ptr)
-  {
-  return ((!ptr) ? 0 : (OFFSET)((char *)ptr - (char *)memp));
-  } /* end of MakeOff */
-#endif
 /*--------------------- End of PLUGUTIL program -----------------------*/

storage/connect/tabjson.cpp

@@ -52,19 +52,10 @@
 /*  External functions.                                                */
 /***********************************************************************/
 USETEMP UseTemp(void);
+bool    JsonAllPath(void);
+int     GetDefaultDepth(void);
 char   *GetJsonNull(void);
 
-//typedef struct _jncol {
-//  struct _jncol *Next;
-//  char *Name;
-//  char *Fmt;
-//  int   Type;
-//  int   Len;
-//  int   Scale;
-//  bool  Cbn;
-//  bool  Found;
-//} JCOL, *PJCL;
-
 /***********************************************************************/
 /* JSONColumns: construct the result blocks containing the description */
 /* of all the columns of a table contained inside a JSON file.         */
@@ -167,23 +158,20 @@ JSONDISC::JSONDISC(PGLOBAL g, uint *lg)
   jsp = NULL;
   row = NULL;
   sep = NULL;
-  i = n = bf = ncol = lvl = 0;
-  all = false;
+  i = n = bf = ncol = lvl = sz = 0;
+  all = strfy = false;
 } // end of JSONDISC constructor
 
 int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
 {
   char filename[_MAX_PATH];
   bool mgo = (GetTypeID(topt->type) == TAB_MONGO);
-  PCSZ level = GetStringTableOption(g, topt, "Level", NULL);
-
-  if (level) {
-    lvl = atoi(level);
-    lvl = (lvl > 16) ? 16 : lvl;
-  } else
-    lvl = 0;
 
+	lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth());
+	lvl = GetIntegerTableOption(g, topt, "Depth", lvl);
 	sep = GetStringTableOption(g, topt, "Separator", ".");
+	sz = GetIntegerTableOption(g, topt, "Jsize", 1024);
+	strfy = GetBooleanTableOption(g, topt, "Stringify", false);
 
   /*********************************************************************/
   /*  Open the input file.                                             */
@@ -254,13 +242,15 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
 
     jsp = (tjsp->GetDoc()) ? tjsp->GetDoc()->GetValue(0) : NULL;
   } else {
-    if (!(tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0)))
+		if (!((tdp->Lrecl = GetIntegerTableOption(g, topt, "Lrecl", 0)))) {
 			if (!mgo) {
 				sprintf(g->Message, "LRECL must be specified for pretty=%d", tdp->Pretty);
 				return 0;
 			} else
 				tdp->Lrecl = 8192;       // Should be enough
 
+		} // endif Lrecl
+
     tdp->Ending = GetIntegerTableOption(g, topt, "Ending", CRLF);
 
     if (tdp->Zipped) {
@@ -304,7 +294,7 @@ int JSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
     // Allocate the parse work memory
     PGLOBAL G = (PGLOBAL)PlugSubAlloc(g, NULL, sizeof(GLOBAL));
     memset(G, 0, sizeof(GLOBAL));
-    G->Sarea_Size = tdp->Lrecl * 10;
+    G->Sarea_Size = (size_t)tdp->Lrecl * 10;
     G->Sarea = PlugSubAlloc(g, NULL, G->Sarea_Size);
     PlugSubSet(G->Sarea, G->Sarea_Size);
     G->jump_level = 0;
@@ -401,6 +391,9 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j)
   PJAR   jar;
 
   if ((valp = jvp ? jvp->GetValue() : NULL)) {
+		if (JsonAllPath() && !fmt[bf])
+			strcat(fmt, colname);
+
 		jcol.Type = valp->GetType();
     jcol.Len = valp->GetValLen();
     jcol.Scale = valp->GetValPrec();
@@ -480,8 +473,16 @@ bool JSONDISC::Find(PGLOBAL g, PJVAL jvp, PCSZ key, int j)
     } // endswitch Type
 
   } else if (lvl >= 0) {
+		if (strfy) {
+			if (!fmt[bf])
+				strcat(fmt, colname);
+
+			strcat(fmt, ".*");
+		}	else if (JsonAllPath() && !fmt[bf])
+			strcat(fmt, colname);
+
 		jcol.Type = TYPE_STRING;
-    jcol.Len = 256;
+    jcol.Len = sz;
     jcol.Scale = 0;
     jcol.Cbn = true;
   } else
@@ -1329,7 +1330,7 @@ bool JSONCOL::ParseJpath(PGLOBAL g)
 {
   char *p, *p1 = NULL, *p2 = NULL, *pbuf = NULL;
   int   i;
-  bool  a, mul = false;
+  bool  a;
 
   if (Parsed)
     return false;                       // Already done
@@ -1486,6 +1487,17 @@ PVAL JSONCOL::MakeJson(PGLOBAL g, PJSON jsp)
   if (Value->IsTypeNum()) {
     strcpy(g->Message, "Cannot make Json for a numeric column");
     Value->Reset();
+	} else if (Value->GetType() == TYPE_BIN) {
+		if ((unsigned)Value->GetClen() >= sizeof(BSON)) {
+			ulong len = Tjp->Lrecl ? Tjp->Lrecl : 500;
+			PBSON bsp = JbinAlloc(g, NULL, len, jsp);
+
+			strcat(bsp->Msg, " column");
+			((BINVAL*)Value)->SetBinValue(bsp, sizeof(BSON));
+		} else {
+			strcpy(g->Message, "Column size too small");
+			Value->SetValue_char(NULL, 0);
+		} // endif Clen
 	}	else
     Value->SetValue_psz(Serialize(g, jsp, NULL, 0));
 
@@ -1559,7 +1571,6 @@ void JSONCOL::ReadColumn(PGLOBAL g)
 PVAL JSONCOL::GetColumnValue(PGLOBAL g, PJSON row, int i)
   {
   int   n = Nod - 1;
-  bool  expd = false;
   PJAR  arp;
   PJVAL val = NULL;
 
@@ -1983,8 +1994,9 @@ int TDBJSON::MakeNewDoc(PGLOBAL g)
 /***********************************************************************/
 int TDBJSON::MakeDocument(PGLOBAL g)
   {
-  char   *p, *memory, *objpath, *key = NULL;
+  char   *p, *p1, *p2, *memory, *objpath, *key = NULL;
   int     len, i = 0;
+	my_bool a;
   MODE    mode = Mode;
   PJSON   jsp;
   PJOB    objp = NULL;
@@ -2027,22 +2039,39 @@ int TDBJSON::MakeDocument(PGLOBAL g)
   if ((objpath = PlugDup(g, Objname))) {
     if (*objpath == '$') objpath++;
     if (*objpath == '.') objpath++;
+		p1 = (*objpath == '[') ? objpath++ : NULL;
 
     /*********************************************************************/
     /*  Find the table in the tree structure.                            */
     /*********************************************************************/
-    for (; jsp && objpath; objpath = p) {
-      if ((p = strchr(objpath, Sep)))
-        *p++ = 0;
+    for (p = objpath; jsp && p; p = (p2 ? p2 : NULL)) {
+			a = (p1 != NULL);
+			p1 = strchr(p, '[');
+			p2 = strchr(p, '.');
 
-      if (*objpath != '[' && !IsNum(objpath)) {
-        // objpass is a key
+			if (!p2)
+				p2 = p1;
+			else if (p1) {
+				if (p1 < p2)
+					p2 = p1;
+				else if (p1 == p2 + 1)
+					*p2++ = 0;		 // Old syntax .[
+				else
+					p1 = NULL;
+
+			}	// endif p1
+
+			if (p2)
+				*p2++ = 0;
+
+      if (!a && *p && *p != '[' && !IsNum(p)) {
+        // obj is a key
         if (jsp->GetType() != TYPE_JOB) {
           strcpy(g->Message, "Table path does not match the json file");
           return RC_FX;
         } // endif Type
 
-        key = objpath;
+        key = p;
         objp = jsp->GetObject();
         arp = NULL;
         val = objp->GetValue(key);
@@ -2053,15 +2082,15 @@ int TDBJSON::MakeDocument(PGLOBAL g)
         } // endif val
 
       } else {
-        if (*objpath == '[') {
+        if (*p == '[') {
           // Old style
-          if (objpath[strlen(objpath) - 1] != ']') {
-            sprintf(g->Message, "Invalid Table path %s", Objname);
+          if (p[strlen(p) - 1] != ']') {
+            sprintf(g->Message, "Invalid Table path near %s", p);
             return RC_FX;
           } else
-            objpath++;
+            p++;
 
-        } // endif objpath
+        } // endif p
 
         if (jsp->GetType() != TYPE_JAR) {
           strcpy(g->Message, "Table path does not match the json file");
@@ -2070,7 +2099,7 @@ int TDBJSON::MakeDocument(PGLOBAL g)
 
         arp = jsp->GetArray();
         objp = NULL;
-        i = atoi(objpath) - B;
+        i = atoi(p) - B;
         val = arp->GetValue(i);
 
         if (!val) {
@@ -2081,7 +2110,7 @@ int TDBJSON::MakeDocument(PGLOBAL g)
       } // endif
 
       jsp = val->GetJson();
-    } // endfor objpath
+    } // endfor p
 
   } // endif objpath
 
@@ -2119,7 +2148,7 @@ int TDBJSON::Cardinality(PGLOBAL g)
   {
   if (!g)
     return (Xcol || Multiple) ? 0 : 1;
-  else if (Cardinal < 0)
+	else if (Cardinal < 0) {
 		if (!Multiple) {
 			if (MakeDocument(g) == RC_OK)
 				Cardinal = Doc->size();
@@ -2127,6 +2156,8 @@ int TDBJSON::Cardinality(PGLOBAL g)
 		} else
 			return 10;
 
+	} // endif Cardinal
+
   return Cardinal;
   } // end of Cardinality
 

storage/connect/tabjson.h

@@ -68,8 +68,8 @@ class JSONDISC : public BLOCK {
 	PCSZ    sep;
 	char    colname[65], fmt[129], buf[16];
 	uint   *length;
-	int     i, n, bf, ncol, lvl;
-	bool    all;
+	int     i, n, bf, ncol, lvl, sz;
+	bool    all, strfy;
 }; // end of JSONDISC
 
 /***********************************************************************/

storage/connect/tabrest.cpp

@@ -158,16 +158,32 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info)
 
   http = GetStringTableOption(g, tp, "Http", NULL);
   uri = GetStringTableOption(g, tp, "Uri", NULL);
-  fn = GetStringTableOption(g, tp, "Filename", "rest.json");
 #if defined(MARIADB)
   ftype = GetStringTableOption(g, tp, "Type", "JSON");
 #else   // !MARIADB
   // OEM tables must specify the file type
   ftype = GetStringTableOption(g, tp, "Ftype", "JSON");
 #endif  // !MARIADB
+	fn = GetStringTableOption(g, tp, "Filename", NULL);
+
+	if (!fn) {
+		int n, m = strlen(ftype) + 1;
+
+		strcat(strcpy(filename, tab), ".");
+		n = strlen(filename);
+
+		// Fold ftype to lower case
+		for (int i = 0; i < m; i++)
+			filename[n + i] = tolower(ftype[i]);
+
+		fn = filename;
+		tp->filename = PlugDup(g, fn);
+	}	// endif fn
 
   //  We used the file name relative to recorded datapath
-  snprintf(filename, sizeof filename, IF_WIN(".\\%s\\%s","./%s/%s"), db, fn);
+	PlugSetPath(filename, fn, db);
+	//strcat(strcat(strcat(strcpy(filename, "."), slash), db), slash);
+  //strncat(filename, fn, _MAX_PATH - strlen(filename));
 
   // Retrieve the file from the web and copy it locally
 	if (http && grf(g->Message, trace(515), http, uri, filename)) {
@@ -226,12 +242,10 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff)
 
   Http = GetStringCatInfo(g, "Http", NULL);
   Uri = GetStringCatInfo(g, "Uri", NULL);
-  Fn = GetStringCatInfo(g, "Filename", "rest.json");
+  Fn = GetStringCatInfo(g, "Filename", NULL);
 
   //  We used the file name relative to recorded datapath
-  //PlugSetPath(filename, Fn, GetPath());
-  strcpy(filename, GetPath());
-	strncat(filename, Fn, _MAX_PATH - strlen(filename));
+  PlugSetPath(filename, Fn, GetPath());
 
   // Retrieve the file from the web and copy it locally
 	rc = grf(g->Message, xt, Http, Uri, filename);
@@ -269,7 +283,7 @@ PTDB RESTDEF::GetTable(PGLOBAL g, MODE m)
   if (trace(515))
     htrc("REST GetTable mode=%d\n", m);
 
-  if (m != MODE_READ && m != MODE_READX) {
+  if (m != MODE_READ && m != MODE_READX && m != MODE_ANY) {
     strcpy(g->Message, "REST tables are currently read only");
     return NULL;
   } // endif m

storage/connect/tabxml.cpp

@@ -3,7 +3,7 @@
 /* -------------                                                       */
 /*  Version 3.0                                                        */
 /*                                                                     */
-/*  Author Olivier BERTRAND          2007 - 2017                       */
+/*  Author Olivier BERTRAND          2007 - 2020                       */
 /*                                                                     */
 /*  This program are the XML tables classes using MS-DOM or libxml2.   */
 /***********************************************************************/
@@ -62,6 +62,8 @@ extern "C" char version[];
 #define TYPE_UNKNOWN     12        /* Must be greater than other types */
 #define XLEN(M)  sizeof(M) - strlen(M) - 1	       /* To avoid overflow*/
 
+int GetDefaultDepth(void);
+
 /***********************************************************************/
 /* Class and structure used by XMLColumns.                             */
 /***********************************************************************/
@@ -149,7 +151,8 @@ PQRYRES XMLColumns(PGLOBAL g, char *db, char *tab, PTOS topt, bool info)
     strcpy(g->Message, MSG(MISSING_FNAME));
     return NULL;
   } else {
-    lvl = GetIntegerTableOption(g, topt, "Level", 0);
+    lvl = GetIntegerTableOption(g, topt, "Level", GetDefaultDepth());
+		lvl = GetIntegerTableOption(g, topt, "Depth", lvl);
 		lvl = (lvl < 0) ? 0 : (lvl > 16) ? 16 : lvl;
   } // endif fn
 

storage/connect/user_connect.cc

@@ -28,7 +28,7 @@
 */
 
 /****************************************************************************/
-/*  Author: Olivier Bertrand  --  bertrandop@gmail.com  --  2004-2015       */
+/*  Author: Olivier Bertrand  --  bertrandop@gmail.com  --  2004-2020       */
 /****************************************************************************/
 #ifdef USE_PRAGMA_IMPLEMENTATION
 #pragma implementation        // gcc: Class implementation
@@ -58,8 +58,8 @@ PCONNECT user_connect::to_users= NULL;
 /****************************************************************************/
 /*  Get the work_size SESSION variable value .                              */
 /****************************************************************************/
-uint GetWorkSize(void);
-void SetWorkSize(uint);
+size_t GetWorkSize(void);
+void  SetWorkSize(size_t);
 
 /* -------------------------- class user_connect -------------------------- */
 
@@ -97,14 +97,14 @@ user_connect::~user_connect()
 bool user_connect::user_init()
 {
   // Initialize Plug-like environment
-  uint      worksize= GetWorkSize();
+  size_t    worksize= GetWorkSize();
   PACTIVITY ap= NULL;
   PDBUSER   dup= NULL;
 
   // Areasize= 64M because of VEC tables. Should be parameterisable
 //g= PlugInit(NULL, 67108864);
 //g= PlugInit(NULL, 134217728);  // 128M was because of old embedded tests
-  g= PlugInit(NULL, worksize);
+  g= PlugInit(NULL, (size_t)worksize);
 
   // Check whether the initialization is complete
   if (!g || !g->Sarea || PlugSubSet(g->Sarea, g->Sarea_Size)
@@ -157,16 +157,17 @@ void user_connect::SetHandler(ha_connect *hc)
 bool user_connect::CheckCleanup(bool force)
 {
   if (thdp->query_id > last_query_id || force) {
-    uint worksize= GetWorkSize(), size = g->Sarea_Size;
+		size_t worksize = GetWorkSize();
 
     PlugCleanup(g, true);
 
-    if (size != worksize) {
+    if (worksize != g->Sarea_Size) {
 			FreeSarea(g);
+			g->Saved_Size = g->Sarea_Size;
 
       // Check whether the work area could be allocated
       if (AllocSarea(g, worksize)) {
-				AllocSarea(g, size);
+				AllocSarea(g, g->Saved_Size);
         SetWorkSize(g->Sarea_Size);       // Was too big
       } // endif sarea
 
@@ -174,10 +175,11 @@ bool user_connect::CheckCleanup(bool force)
 
     PlugSubSet(g->Sarea, g->Sarea_Size);
     g->Xchk = NULL;
-    g->Createas = 0;
+    g->Createas = false;
     g->Alchecked = 0;
     g->Mrr = 0;
 		g->More = 0;
+		g->Saved_Size = 0;
 		last_query_id= thdp->query_id;
 
     if (trace(65) && !force)

storage/connect/value.cpp

@@ -2250,6 +2250,15 @@ void BINVAL::SetBinValue(void *p)
 	Len = Clen;
 } // end of SetBinValue
 
+/***********************************************************************/
+/*  BINVAL SetBinValue: fill string with len bytes.                    */
+/***********************************************************************/
+void BINVAL::SetBinValue(void* p, ulong len)
+{
+	memcpy(Binp, p, len);
+	Len = len;
+} // end of SetBinValue
+
 /***********************************************************************/
 /*  GetBinValue: fill a buffer with the internal binary value.         */
 /*  This function checks whether the buffer length is enough and       */

storage/connect/value.h

@@ -115,7 +115,7 @@ class DllExport VALUE : public BLOCK {
   virtual void   SetValue(ulonglong) {assert(false);}
   virtual void   SetValue(double) {assert(false);}
   virtual void   SetValue_pvblk(PVBLK blk, int n) = 0;
-  virtual void   SetBinValue(void *p) = 0;
+	virtual void   SetBinValue(void* p) = 0;
 	virtual bool   GetBinValue(void *buf, int buflen, bool go) = 0;
   virtual int    ShowValue(char *buf, int len) = 0;
   virtual char  *GetCharString(char *p) = 0;
@@ -385,6 +385,7 @@ class DllExport BINVAL: public VALUE {
   virtual void   SetValue(ulonglong n);
   virtual void   SetValue(double f);
   virtual void   SetBinValue(void *p);
+	virtual void   SetBinValue(void* p, ulong len);
 	virtual bool   GetBinValue(void *buf, int buflen, bool go);
   virtual int    CompareValue(PVAL) {assert(false); return 0;}
   virtual int    ShowValue(char *buf, int len);