=====================================================================

This new CONNECT version 1.07 fully implements NOSQL support.
It allows working on JSON or XML data retrieved as REST query results
from all binary distributions of MariaDB when cpprestsdk is installed
and the GetRest library is available.
=====================================================================
- Make Rest available for MariaDB binary distributed versions.
  Change RestGet function so it can be called from a library.
  modified:   storage/connect/CMakeLists.txt
  modified:   storage/connect/restget.cpp
  modified:   storage/connect/tabrest.cpp

- Make column FLAG option available to discovery functions.
  modified:   storage/connect/ha_connect.cc
  modified:   storage/connect/plgdbsem.h

- Update CONNECT version number and date.
  modified:   storage/connect/ha_connect.cc

- Move OEMColumns function from mycat.cc to reldef.cpp.
  modified:   storage/connect/mycat.cc
  modified:   storage/connect/reldef.cpp

- Allocate tables as TABREF (was RELDEF)
  modified:   storage/connect/mycat.cc
  modified:   storage/connect/mycat.h

- Fix MDEV-20845 by commenting out TIMEOUT setting.
  modified:   storage/connect/myconn.cpp

- Call DefineAM before calling GetColCatInfo. Column offset
  is now based on record format instead of table type.
  The RECFM_VCT format was added.
  This enables tables to specify the record format and is
  useful in particular for OEM tables.
  modified:   storage/connect/plgdbsem.h
  modified:   storage/connect/reldef.cpp
  modified:   storage/connect/reldef.h
  modified:   storage/connect/tabdos.cpp
  modified:   storage/connect/tabdos.h
  modified:   storage/connect/tabfix.cpp
  modified:   storage/connect/tabfmt.cpp
  modified:   storage/connect/tabmysql.cpp
  modified:   storage/connect/tabutil.cpp
  modified:   storage/connect/tabutil.h
  modified:   storage/connect/tabvct.cpp
  modified:   storage/connect/xindex.cpp

storage/connect/CMakeLists.txt

@@ -312,6 +312,8 @@ OPTION(CONNECT_WITH_REST "Compile CONNECT storage engine with REST support" ON)
 
 IF(CONNECT_WITH_REST)
   MESSAGE(STATUS "=====> REST support is ON")
+  SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp tabrest.h)
+  add_definitions(-DREST_SUPPORT)
   FIND_PACKAGE(cpprestsdk)
   IF (cpprestsdk_FOUND)
     MESSAGE(STATUS "=====> cpprestsdk found")
@@ -326,8 +328,8 @@ IF(CONNECT_WITH_REST)
 #     Comment it out if not needed depending on your cpprestsdk installation.
 	  SET(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /MDd")
 	ENDIF(UNIX)
-    SET(CONNECT_SOURCES ${CONNECT_SOURCES} tabrest.cpp restget.cpp tabrest.h)
-    add_definitions(-DREST_SUPPORT)
+    SET(CONNECT_SOURCES ${CONNECT_SOURCES} restget.cpp)
+    add_definitions(-DREST_SOURCE)
   ELSE(NOT cpprestsdk_FOUND)
     MESSAGE(STATUS "=====> cpprestsdk package not found")
   ENDIF (cpprestsdk_FOUND)

storage/connect/ha_connect.cc

@@ -170,9 +170,9 @@
 #define JSONMAX      10             // JSON Default max grp size
 
 extern "C" {
-       char version[]= "Version 1.06.0010 June 01, 2019";
+       char version[]= "Version 1.07.0001 November 12, 2019";
 #if defined(__WIN__)
-       char compver[]= "Version 1.06.0010 " __DATE__ " "  __TIME__;
+       char compver[]= "Version 1.07.0001 " __DATE__ " "  __TIME__;
        char slash= '\\';
 #else   // !__WIN__
        char slash= '/';
@@ -6062,7 +6062,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd,
 				} // endif !nblin
 
 				for (i= 0; !rc && i < qrp->Nblin; i++) {
-					typ= len= prec= dec= 0;
+					typ= len= prec= dec= flg= 0;
 					tm= NOT_NULL_FLAG;
 					cnm= (char*)"noname";
 					dft= xtra= key= fmt= tn= NULL;
@@ -6102,6 +6102,9 @@ static int connect_assisted_discovery(handlerton *, THD* thd,
 								if (crp->Kdata->GetIntValue(i))
 									tm= 0;               // Nullable
 
+								break;
+							case FLD_FLAG:
+								flg = crp->Kdata->GetIntValue(i);
 								break;
 							case FLD_FORMAT:
 								fmt= (crp->Kdata) ? crp->Kdata->GetCharValue(i) : NULL;
@@ -6233,7 +6236,7 @@ static int connect_assisted_discovery(handlerton *, THD* thd,
 
 						// Now add the field
 						if (add_field(&sql, cnm, typ, prec, dec, key, tm, rem, dft, xtra,
-							fmt, 0, dbf, v))
+							fmt, flg, dbf, v))
 							rc= HA_ERR_OUT_OF_MEM;
 				} // endfor i
 
@@ -7379,14 +7382,14 @@ maria_declare_plugin(connect)
   &connect_storage_engine,
   "CONNECT",
   "Olivier Bertrand",
-  "Management of External Data (SQL/NOSQL/MED), including many file formats",
+  "Management of External Data (SQL/NOSQL/MED), including Rest query results",
   PLUGIN_LICENSE_GPL,
   connect_init_func,                            /* Plugin Init */
   connect_done_func,                            /* Plugin Deinit */
-  0x0106,                                       /* version number (1.06) */
+  0x0107,                                       /* version number (1.07) */
   NULL,                                         /* status variables */
   connect_system_variables,                     /* system variables */
-  "1.06.0010",                                  /* string version */
+  "1.07.0001",                                  /* string version */
 	MariaDB_PLUGIN_MATURITY_STABLE                /* maturity */
 }
 maria_declare_plugin_end;

storage/connect/mycat.cc

@@ -95,7 +95,7 @@
 #endif   // ZIP_SUPPORT
 #if defined(REST_SUPPORT)
 #include "tabrest.h"
-#endif   // Rest_SUPPORT
+#endif   // REST_SUPPORT
 #include "mycat.h"
 
 /***********************************************************************/
@@ -107,7 +107,6 @@ extern "C" HINSTANCE s_hModule;           // Saved module handle
 #if defined(JAVA_SUPPORT) || defined(CMGO_SUPPORT)
 bool MongoEnabled(void);
 #endif   // JAVA_SUPPORT || CMGO_SUPPORT
-PQRYRES OEMColumns(PGLOBAL g, PTOS topt, char *tab, char *db, bool info);
 
 /***********************************************************************/
 /*  Get the plugin directory.                                          */
@@ -349,100 +348,6 @@ uint GetFuncID(const char *func)
   return fnc;
   } // end of GetFuncID
 
-/***********************************************************************/
-/*  OEMColumn: Get table column info for an OEM table.                 */
-/***********************************************************************/
-PQRYRES OEMColumns(PGLOBAL g, PTOS topt, char *tab, char *db, bool info)
-  {
-  typedef PQRYRES (__stdcall *XCOLDEF) (PGLOBAL, void*, char*, char*, bool);
-  const char *module, *subtype;
-  char    c, soname[_MAX_PATH], getname[40] = "Col";
-#if defined(__WIN__)
-  HANDLE  hdll;               /* Handle to the external DLL            */
-#else   // !__WIN__
-  void   *hdll;               /* Handle for the loaded shared library  */
-#endif  // !__WIN__
-  XCOLDEF coldef = NULL;
-  PQRYRES qrp = NULL;
-
-  module = topt->module;
-  subtype = topt->subtype;
-
-  if (!module || !subtype)
-    return NULL;
-
-  /*********************************************************************/
-  /*  Ensure that the .dll doesn't have a path.                        */
-  /*  This is done to ensure that only approved dll from the system    */
-  /*  directories are used (to make this even remotely secure).        */
-  /*********************************************************************/
-  if (check_valid_path(module, strlen(module))) {
-    strcpy(g->Message, "Module cannot contain a path");
-    return NULL;
-  } else
-    PlugSetPath(soname, module, GetPluginDir());
-
-  // The exported name is always in uppercase
-  for (int i = 0; ; i++) {
-    c = subtype[i];
-    getname[i + 3] = toupper(c);
-    if (!c) break;
-    } // endfor i
-
-#if defined(__WIN__)
-  // Load the Dll implementing the table
-  if (!(hdll = LoadLibrary(soname))) {
-    char  buf[256];
-    DWORD rc = GetLastError();
-
-    sprintf(g->Message, MSG(DLL_LOAD_ERROR), rc, soname);
-    FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM |
-                  FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0,
-                  (LPTSTR)buf, sizeof(buf), NULL);
-    strcat(strcat(g->Message, ": "), buf);
-    return NULL;
-    } // endif hDll
-
-  // Get the function returning an instance of the external DEF class
-  if (!(coldef = (XCOLDEF)GetProcAddress((HINSTANCE)hdll, getname))) {
-    sprintf(g->Message, MSG(PROCADD_ERROR), GetLastError(), getname);
-    FreeLibrary((HMODULE)hdll);
-    return NULL;
-    } // endif coldef
-#else   // !__WIN__
-  const char *error = NULL;
-
-  // Load the desired shared library
-  if (!(hdll = dlopen(soname, RTLD_LAZY))) {
-    error = dlerror();
-    sprintf(g->Message, MSG(SHARED_LIB_ERR), soname, SVP(error));
-    return NULL;
-    } // endif Hdll
-
-  // Get the function returning an instance of the external DEF class
-  if (!(coldef = (XCOLDEF)dlsym(hdll, getname))) {
-    error = dlerror();
-    sprintf(g->Message, MSG(GET_FUNC_ERR), getname, SVP(error));
-    dlclose(hdll);
-    return NULL;
-    } // endif coldef
-#endif  // !__WIN__
-
-  // Just in case the external Get function does not set error messages
-  sprintf(g->Message, "Error getting column info from %s", subtype);
-
-  // Get the table column definition
-  qrp = coldef(g, topt, tab, db, info);
-
-#if defined(__WIN__)
-  FreeLibrary((HMODULE)hdll);
-#else   // !__WIN__
-  dlclose(hdll);
-#endif  // !__WIN__
-
-  return qrp;
-  } // end of OEMColumns
-
 /* ------------------------- Class CATALOG --------------------------- */
 
 /***********************************************************************/
@@ -483,10 +388,10 @@ void MYCAT::Reset(void)
 /*  GetTableDesc: retrieve a table descriptor.                         */
 /*  Look for a table descriptor matching the name and type.            */
 /***********************************************************************/
-PRELDEF MYCAT::GetTableDesc(PGLOBAL g, PTABLE tablep,
+PTABDEF MYCAT::GetTableDesc(PGLOBAL g, PTABLE tablep,
                                        LPCSTR type, PRELDEF *)
 {
-  PRELDEF tdp= NULL;
+  PTABDEF tdp= NULL;
 
   if (trace(1))
     htrc("GetTableDesc: name=%s am=%s\n", tablep->GetName(), SVP(type));
@@ -507,12 +412,12 @@ PRELDEF MYCAT::GetTableDesc(PGLOBAL g, PTABLE tablep,
 /*  MakeTableDesc: make a table/view description.                      */
 /*  Note: caller must check if name already exists before calling it.  */
 /***********************************************************************/
-PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am)
+PTABDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am)
   {
   TABTYPE tc;
   LPCSTR  name= (PSZ)PlugDup(g, tablep->GetName());
   LPCSTR  schema= (PSZ)PlugDup(g, tablep->GetSchema());
-  PRELDEF tdp= NULL;
+  PTABDEF tdp= NULL;
 
   if (trace(1))
     htrc("MakeTableDesc: name=%s schema=%s am=%s\n",
@@ -580,8 +485,8 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am)
     } // endswitch
 
   // Do make the table/view definition
-  if (tdp && tdp->Define(g, this, name, schema, am))
-    tdp= NULL;
+	if (tdp && tdp->Define(g, this, name, schema, am))
+		tdp = NULL;
 
   if (trace(1))
     htrc("Table %s made\n", am);
@@ -594,7 +499,7 @@ PRELDEF MYCAT::MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am)
 /***********************************************************************/
 PTDB MYCAT::GetTable(PGLOBAL g, PTABLE tablep, MODE mode, LPCSTR type)
   {
-  PRELDEF tdp;
+  PTABDEF tdp;
   PTDB    tdbp= NULL;
 //  LPCSTR  name= tablep->GetName();
 

storage/connect/mycat.h

@@ -102,14 +102,14 @@ class MYCAT : public CATALOG {
   // Methods
   void    Reset(void);
   bool    StoreIndex(PGLOBAL, PTABDEF) {return false;}  // Temporary
-	PRELDEF GetTableDesc(PGLOBAL g, PTABLE tablep,
+	PTABDEF GetTableDesc(PGLOBAL g, PTABLE tablep,
 		                   LPCSTR type, PRELDEF *prp = NULL);
   PTDB    GetTable(PGLOBAL g, PTABLE tablep, 
                               MODE mode = MODE_READ, LPCSTR type = NULL);
   void    ClearDB(PGLOBAL g);
 
  protected:
-	PRELDEF MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am);
+	PTABDEF MakeTableDesc(PGLOBAL g, PTABLE tablep, LPCSTR am);
 
   // Members
   ha_connect *Hc;                          // The Connect handler

storage/connect/myconn.cpp

@@ -472,7 +472,7 @@ int MYSQLC::Open(PGLOBAL g, const char *host, const char *db,
                             int pt, const char *csname)
   {
   const char *pipe = NULL;
-  uint        cto = 10, nrt = 20;
+  //uint      cto = 10, nrt = 20;
   my_bool     my_true= 1;
 
   m_DB = mysql_init(NULL);
@@ -485,11 +485,11 @@ int MYSQLC::Open(PGLOBAL g, const char *host, const char *db,
 	if (trace(1))
 		htrc("MYSQLC Open: m_DB=%.4X size=%d\n", m_DB, (int)sizeof(*m_DB));
 
-	// Removed to do like FEDERATED do
+	// Removed to do like FEDERATED does
 //mysql_options(m_DB, MYSQL_READ_DEFAULT_GROUP, "client-mariadb");
-  mysql_options(m_DB, MYSQL_OPT_USE_REMOTE_CONNECTION, NULL);
-  mysql_options(m_DB, MYSQL_OPT_CONNECT_TIMEOUT, &cto);
-  mysql_options(m_DB, MYSQL_OPT_READ_TIMEOUT, &nrt);
+//mysql_options(m_DB, MYSQL_OPT_USE_REMOTE_CONNECTION, NULL);
+//mysql_options(m_DB, MYSQL_OPT_CONNECT_TIMEOUT, &cto);
+//mysql_options(m_DB, MYSQL_OPT_READ_TIMEOUT, &nrt);
 //mysql_options(m_DB, MYSQL_OPT_WRITE_TIMEOUT, ...);
 
 #if defined(__WIN__)

storage/connect/plgdbsem.h

@@ -149,16 +149,22 @@ enum AMT {TYPE_AM_ERROR =   0,        /* Type not defined              */
 	        TYPE_AM_MGO   = 194,				/* MGO access method type no     */
 					TYPE_AM_OUT   = 200};       /* Output relations (storage)    */
 
-enum RECFM {RECFM_NAF   =    -2,      /* Not a file                    */
-            RECFM_OEM   =    -1,      /* OEM file access method        */
-            RECFM_VAR   =     0,      /* Varying length DOS files      */
-            RECFM_FIX   =     1,      /* Fixed length DOS files        */
-            RECFM_BIN   =     2,      /* Binary DOS files (also fixed) */
-            RECFM_VCT   =     3,      /* VCT formatted files           */
-            RECFM_ODBC  =     4,      /* Table accessed via ODBC       */
-						RECFM_JDBC  =     5,      /* Table accessed via JDBC       */
-						RECFM_PLG   =     6,      /* Table accessed via PLGconn    */
-            RECFM_DBF   =     7};     /* DBase formatted file          */
+enum RECFM {RECFM_DFLT  =     0,      /* Default table type            */
+            RECFM_NAF   =     1,      /* Not a file table              */
+            RECFM_OEM   =     2,      /* OEM table                     */
+            RECFM_VAR   =     3,      /* Varying length DOS files      */
+            RECFM_FIX   =     4,      /* Fixed length DOS files        */
+            RECFM_BIN   =     5,      /* Binary DOS files (also fixed) */
+						RECFM_DBF   =     6,      /* DBase formatted file          */
+						RECFM_CSV   =     7,      /* CSV file                      */
+						RECFM_FMT   =     8,      /* FMT formatted file            */
+						RECFM_VCT   =     9,      /* VCT formatted files           */
+						RECFM_XML   =    10,      /* XML formatted files           */
+						RECFM_JASON =    11,      /* JASON formatted files         */
+						RECFM_DIR   =    12,      /* DIR table                     */
+						RECFM_ODBC  =    13,      /* Table accessed via ODBC       */
+						RECFM_JDBC  =    14,      /* Table accessed via JDBC       */
+						RECFM_PLG   =    15};     /* Table accessed via PLGconn    */
 
 enum MISC {DB_TABNO     =     1,      /* DB routines in Utility Table  */
            MAX_MULT_KEY =    10,      /* Max multiple key number       */
@@ -537,7 +543,8 @@ enum XFLD {FLD_NO       =  0,         /* Not a field definition item   */
            FLD_FORMAT   = 16,         /* Field format                  */
            FLD_CAT      = 17,         /* Table catalog                 */
            FLD_SCHEM    = 18,         /* Table schema                  */
-           FLD_TABNAME  = 19};        /* Column Table name             */
+           FLD_TABNAME  = 19,         /* Column Table name             */
+					 FLD_FLAG     = 20};        /* Field flag (CONNECT specific) */
 
 /***********************************************************************/
 /*  Result of last SQL noconv query.                                   */

storage/connect/reldef.h

@@ -84,10 +84,12 @@ class DllExport TABDEF : public RELDEF {   /* Logical table descriptor */
   void    SetNext(PTABDEF tdfp) {Next = tdfp;}
   int     GetMultiple(void) {return Multiple;}
   int     GetPseudo(void) {return Pseudo;}
-  PCSZ    GetPath(void);
+	RECFM   GetRecfm(void) {return Recfm;}
+	PCSZ    GetPath(void);
 //PSZ     GetPath(void)
 //          {return (Database) ? (PSZ)Database : Cat->GetDataPath();}
-  bool    SepIndex(void) {return GetBoolCatInfo("SepIndex", false);}
+	RECFM   GetTableFormat(const char* type);
+	bool    SepIndex(void) {return GetBoolCatInfo("SepIndex", false);}
   bool    IsReadOnly(void) {return Read_Only;}
   virtual AMT    GetDefType(void) {return TYPE_AM_TAB;}
   virtual PIXDEF GetIndx(void) {return NULL;}
@@ -108,7 +110,8 @@ class DllExport TABDEF : public RELDEF {   /* Logical table descriptor */
   // Members
   PCSZ    Schema;               /* Table schema (for ODBC)             */
   PCSZ    Desc;                 /* Table description                   */
-  uint    Catfunc;              /* Catalog function ID                 */
+	RECFM   Recfm;                /* File or table format                */
+	uint    Catfunc;              /* Catalog function ID                 */
   int     Card;                 /* (max) number of rows in table       */
   int     Elemt;                /* Number of rows in blocks or rowset  */
   int     Sort;                 /* Table already sorted ???            */

storage/connect/restget.cpp

@@ -4,12 +4,6 @@
 /***********************************************************************/
 #include <cpprest/filestream.h>
 #include <cpprest/http_client.h>
-#if defined(MARIADB)
-#include <my_global.h>
-#else
-#include "mini-global.h"
-#define _OS_H_INCLUDED     // Prevent os.h to be called
-#endif
 
 using namespace utility::conversions; // String conversions utilities
 using namespace web;                  // Common features like URIs.
@@ -17,24 +11,24 @@ using namespace web::http;            // Common HTTP functionality
 using namespace web::http::client;    // HTTP client features
 using namespace concurrency::streams; // Asynchronous streams
 
-#include "global.h"
+typedef const char* PCSZ;
 
 /***********************************************************************/
 /*  Make a local copy of the requested file.                           */
 /***********************************************************************/
-int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn)
+int restGetFile(char *m, bool xt, PCSZ http, PCSZ uri, PCSZ fn)
 {
   int  rc = 0;
-	bool xt = trace(515);
   auto fileStream = std::make_shared<ostream>();
 
   if (!http || !fn) {
-    strcpy(g->Message, "Missing http or filename");
-    return 2;
+    //strcpy(g->Message, "Missing http or filename");
+		strcpy(m, "Missing http or filename");
+		return 2;
   } // endif
 
 	if (xt)
-	  htrc("restGetFile: fn=%s\n", fn);
+		fprintf(stderr, "restGetFile: fn=%s\n", fn);
 
   // Open stream to output file.
   pplx::task<void> requestTask = fstream::open_ostream(to_string_t(fn))
@@ -42,7 +36,7 @@ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn)
       *fileStream= outFile;
 
 			if (xt)
-				htrc("Outfile isopen=%d\n", outFile.is_open());
+				fprintf(stderr, "Outfile isopen=%d\n", outFile.is_open());
 
       // Create http_client to send the request.
       http_client client(to_string_t(http));
@@ -58,8 +52,8 @@ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn)
     // Handle response headers arriving.
     .then([=](http_response response) {
 			if (xt)
-				htrc("Received response status code:%u\n",
-                       response.status_code());
+				fprintf(stderr, "Received response status code:%u\n",
+                                  response.status_code());
 
       // Write response body into the file.
       return response.body().read_to_end(fileStream->streambuf());
@@ -68,27 +62,27 @@ int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn)
     // Close the file stream.
     .then([=](size_t n) {
 			if (xt)
-			  htrc("Return size=%u\n", n);
+				fprintf(stderr, "Return size=%zu\n", n);
 
       return fileStream->close();
     });
 
   // Wait for all the outstanding I/O to complete and handle any exceptions
   try {
-    requestTask.wait();
-
 		if (xt)
-      htrc("In Wait\n");
+			fprintf(stderr, "Waiting\n");
 
+		requestTask.wait();
   } catch (const std::exception &e) {
 		if (xt)
-		  htrc("Error exception: %s\n", e.what());
-    sprintf(g->Message, "Error exception: %s", e.what());
-    rc= 1;
+			fprintf(stderr, "Error exception: %s\n", e.what());
+
+		sprintf(m, "Error exception: %s", e.what());
+		rc= 1;
   } // end try/catch
 
 	if (xt)
-	  htrc("restget done: rc=%d\n", rc);
+		fprintf(stderr, "restget done: rc=%d\n", rc);
 
   return rc;
 } // end of restGetFile

storage/connect/tabdos.cpp

@@ -45,7 +45,7 @@
 #include "global.h"
 #include "osutil.h"
 #include "plgdbsem.h"
-#include "catalog.h"
+//#include "catalog.h"
 #include "mycat.h"
 #include "xindex.h"
 #include "filamap.h"
@@ -161,7 +161,12 @@ bool DOSDEF::DefineAM(PGLOBAL g, LPCSTR am, int)
 //Last = GetIntCatInfo("Last", 0);
   Ending = GetIntCatInfo("Ending", CRLF);
 
-  if (Recfm == RECFM_FIX || Recfm == RECFM_BIN) {
+	if (Ending <= 0) {
+		Ending = (Recfm == RECFM_BIN || Recfm == RECFM_VCT) ? 0 : CRLF;
+		SetIntCatInfo("Ending", Ending);
+	} // endif ending
+
+	if (Recfm == RECFM_FIX || Recfm == RECFM_BIN) {
     Huge = GetBoolCatInfo("Huge", Cat->GetDefHuge());
     Padded = GetBoolCatInfo("Padded", false);
     Blksize = GetIntCatInfo("Blksize", 0);
@@ -191,7 +196,8 @@ bool DOSDEF::GetOptFileName(PGLOBAL g, char *filename)
     case RECFM_FIX: ftype = ".fop"; break;
     case RECFM_BIN: ftype = ".bop"; break;
     case RECFM_VCT: ftype = ".vop"; break;
-    case RECFM_DBF: ftype = ".dbp"; break;
+		case RECFM_CSV: ftype = ".cop"; break;
+		case RECFM_DBF: ftype = ".dbp"; break;
     default:
       sprintf(g->Message, MSG(INVALID_FTYPE), Recfm);
       return true;
@@ -261,7 +267,8 @@ bool DOSDEF::DeleteIndexFile(PGLOBAL g, PIXDEF pxdf)
     case RECFM_FIX: ftype = ".fnx"; break;
     case RECFM_BIN: ftype = ".bnx"; break;
     case RECFM_VCT: ftype = ".vnx"; break;
-    case RECFM_DBF: ftype = ".dbx"; break;
+		case RECFM_CSV: ftype = ".cnx"; break;
+		case RECFM_DBF: ftype = ".dbx"; break;
     default:
       sprintf(g->Message, MSG(BAD_RECFM_VAL), Recfm);
       return true;
@@ -2257,7 +2264,7 @@ int TDBDOS::ReadDB(PGLOBAL g)
 /***********************************************************************/
 bool TDBDOS::PrepareWriting(PGLOBAL)
   {
-  if (!Ftype && (Mode == MODE_INSERT || Txfp->GetUseTemp())) {
+  if (Ftype == RECFM_VAR && (Mode == MODE_INSERT || Txfp->GetUseTemp())) {
     char *p;
 
     /*******************************************************************/
@@ -2542,7 +2549,8 @@ void DOSCOL::ReadColumn(PGLOBAL g)
   /*********************************************************************/
   /*  For a variable length file, check if the field exists.           */
   /*********************************************************************/
-  if (tdbp->Ftype == RECFM_VAR && strlen(tdbp->To_Line) < (unsigned)Deplac)
+  if ((tdbp->Ftype == RECFM_VAR || tdbp->Ftype == RECFM_CSV)
+				&& strlen(tdbp->To_Line) < (unsigned)Deplac)
     field = 0;
   else if (Dsp)
     for(i = 0; i < field; i++)
@@ -2552,7 +2560,8 @@ void DOSCOL::ReadColumn(PGLOBAL g)
   switch (tdbp->Ftype) {
     case RECFM_VAR:
     case RECFM_FIX:            // Fixed length text file
-    case RECFM_DBF:            // Fixed length DBase file
+		case RECFM_CSV:            // Variable length CSV or FMT file
+		case RECFM_DBF:            // Fixed length DBase file
       if (Nod) switch (Buf_Type) {
         case TYPE_INT:
         case TYPE_SHORT:

storage/connect/tabdos.h

@@ -80,7 +80,6 @@ class DllExport DOSDEF : public TABDEF {  /* Logical table description */
 	PCSZ    Entry;						  /* Zip entry name or pattern						 */
 	PCSZ    Pwd;						    /* Zip password             						 */
 	PIXDEF  To_Indx;            /* To index definitions blocks           */
-  RECFM   Recfm;              /* 0:VAR, 1:FIX, 2:BIN, 3:VCT, 6:DBF     */
   bool    Mapped;             /* 0: disk file, 1: memory mapped file   */
 	bool    Zipped;             /* true for zipped table file            */
 	bool    Mulentries;         /* true for multiple entries             */

storage/connect/tabfix.cpp

@@ -84,7 +84,7 @@ PTDB TDBFIX::Clone(PTABS t)
 
   tp = new(g) TDBFIX(g, this);
 
-  if (Ftype < 2) {
+  if (Ftype == RECFM_VAR || Ftype == RECFM_FIX) {
     // File is text
     PDOSCOL cp1, cp2;
 

storage/connect/tabfmt.cpp

 /************* TabFmt C++ Program Source Code File (.CPP) **************/
 /* PROGRAM NAME: TABFMT                                                */
 /* -------------                                                       */
-/*  Version 3.9.2                                                      */
+/*  Version 3.9.3                                                      */
 /*                                                                     */
 /* COPYRIGHT:                                                          */
 /* ----------                                                          */
-/*  (C) Copyright to the author Olivier BERTRAND          2001 - 2017  */
+/*  (C) Copyright to the author Olivier BERTRAND          2001 - 2019  */
 /*                                                                     */
 /* WHAT THIS PROGRAM DOES:                                             */
 /* -----------------------                                             */
@@ -477,6 +477,7 @@ bool CSVDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff)
   if (DOSDEF::DefineAM(g, "CSV", poff))
     return true;
 
+	Recfm = RECFM_CSV;
   GetCharCatInfo("Separator", ",", buf, sizeof(buf));
   Sep = (strlen(buf) == 2 && buf[0] == '\\' && buf[1] == 't') ? '\t' : *buf;
   Quoted = GetIntCatInfo("Quoted", -1);

storage/connect/tabmysql.cpp

@@ -342,11 +342,13 @@ bool MYSQLDEF::DefineAM(PGLOBAL g, LPCSTR am, int)
     Delayed = !!GetIntCatInfo("Delayed", 0);
   } else {
     // MYSQL access from a PROXY table 
-    Tabschema = GetStringCatInfo(g, "Database", Tabschema ? Tabschema : PlugDup(g, "*"));
+		TABLE_SHARE* s;
+
+		Tabschema = GetStringCatInfo(g, "Database", Tabschema ? Tabschema : PlugDup(g, "*"));
     Isview = GetBoolCatInfo("View", false);
 
     // We must get other connection parms from the calling table
-    Remove_tshp(Cat);
+    s = Remove_tshp(Cat);
     url = GetStringCatInfo(g, "Connect", NULL);
 
     if (!url || !*url) { 
@@ -365,6 +367,9 @@ bool MYSQLDEF::DefineAM(PGLOBAL g, LPCSTR am, int)
     } // endif url
 
     Tabname = Name;
+
+		// Needed for column description
+		Restore_tshp(Cat, s);
   } // endif am
 
   if ((Srcdef = GetStringCatInfo(g, "Srcdef", NULL))) {

storage/connect/tabrest.cpp

 /*************** Rest C++ Program Source Code File (.CPP) **************/
-/* PROGRAM NAME: Rest   Version 1.5                                    */
+/* PROGRAM NAME: Rest   Version 1.6                                    */
 /*  (C) Copyright to the author Olivier BERTRAND          2018 - 2019  */
 /*  This program is the REST Web API support for MariaDB.              */
 /*  When compiled without MARIADB defined, it is the EOM module code.  */
@@ -39,7 +39,7 @@
 /***********************************************************************/
 /*  Get the file from the Web.                                         */
 /***********************************************************************/
-int restGetFile(PGLOBAL g, PCSZ http, PCSZ uri, PCSZ fn);
+int restGetFile(char *m, bool x, PCSZ http, PCSZ uri, PCSZ fn);
 
 #if defined(__WIN__)
 static PCSZ slash = "\\";
@@ -48,6 +48,10 @@ static PCSZ slash = "/";
 #define stricmp strcasecmp
 #endif // !__WIN__
 
+typedef int(__stdcall* XGETREST) (char*, bool, PCSZ, PCSZ, PCSZ);
+static XGETREST getRestFnc = NULL;
+
+
 #if !defined(MARIADB)
 /***********************************************************************/
 /*  DB static variables.                                               */
@@ -75,6 +79,74 @@ PTABDEF __stdcall GetREST(PGLOBAL g, void *memp)
 } // end of GetREST
 #endif   // !MARIADB
 
+/***********************************************************************/
+/*  GetREST: get the external TABDEF from OEM module.                  */
+/***********************************************************************/
+XGETREST GetRestFunction(PGLOBAL g)
+{
+	if (getRestFnc)
+		return getRestFnc;
+	
+#if !defined(REST_SOURCE)
+	if (trace(515))
+		htrc("Looking for GetRest library\n");
+
+#if defined(__WIN__)
+	HANDLE Hdll;
+	const char* soname = "GetRest.dll";   // Module name
+
+	if (!(Hdll = LoadLibrary(soname))) {
+		char  buf[256];
+		DWORD rc = GetLastError();
+
+		sprintf(g->Message, MSG(DLL_LOAD_ERROR), rc, soname);
+		FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM |
+			FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0,
+			(LPTSTR)buf, sizeof(buf), NULL);
+		strcat(strcat(g->Message, ": "), buf);
+		return NULL;
+	} // endif Hdll
+
+// Get the function returning an instance of the external DEF class
+	if (!(getRestFnc = (XGETREST)GetProcAddress((HINSTANCE)Hdll, "restGetFile"))) {
+		char  buf[256];
+		DWORD rc = GetLastError();
+
+		sprintf(g->Message, MSG(PROCADD_ERROR), rc, "restGetFile");
+		FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM |
+			FORMAT_MESSAGE_IGNORE_INSERTS, NULL, rc, 0,
+			(LPTSTR)buf, sizeof(buf), NULL);
+		strcat(strcat(g->Message, ": "), buf);
+		FreeLibrary((HMODULE)Hdll);
+		return NULL;
+	} // endif getRestFnc
+#else   // !__WIN__
+	void* Hso;
+	const char* error = NULL;
+	const char* soname = "GetRest.so";   // Module name
+
+	// Load the desired shared library
+	if (!(Hso = dlopen(soname, RTLD_LAZY))) {
+		error = dlerror();
+		sprintf(g->Message, MSG(SHARED_LIB_ERR), soname, SVP(error));
+		return NULL;
+	} // endif Hdll
+
+// Get the function returning an instance of the external DEF class
+	if (!(getRestFnc = (XGETREST)dlsym(Hdll, "restGetFile"))) {
+		error = dlerror();
+		sprintf(g->Message, MSG(GET_FUNC_ERR), "restGetFile", SVP(error));
+		dlclose(Hso);
+		return NULL;
+	} // endif getdef
+#endif  // !__WIN__
+#else
+	getRestFnc = restGetFile;
+#endif
+
+	return getRestFnc;
+} // end of GetRestFunction
+
 /***********************************************************************/
 /*  Return the columns definition to MariaDB.                          */
 /***********************************************************************/
@@ -87,6 +159,10 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info)
   PQRYRES qrp= NULL;
   char filename[_MAX_PATH + 1];  // MAX PATH ???
   PCSZ http, uri, fn, ftype;
+	XGETREST grf = GetRestFunction(g);
+
+	if (!grf)
+		return NULL;
 
   http = GetStringTableOption(g, tp, "Http", NULL);
   uri = GetStringTableOption(g, tp, "Uri", NULL);
@@ -103,8 +179,8 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info)
   strncat(filename, fn, _MAX_PATH);
 
   // Retrieve the file from the web and copy it locally
-  if (http && restGetFile(g, http, uri, filename)) {
-    // sprintf(g->Message, "Failed to get file at %s", http);
+	if (http && grf(g->Message, trace(515), http, uri, filename)) {
+			// sprintf(g->Message, "Failed to get file at %s", http);
   } else if (!stricmp(ftype, "XML"))
     qrp = XMLColumns(g, db, tab, tp, info);
   else if (!stricmp(ftype, "JSON"))
@@ -124,9 +200,14 @@ PQRYRES __stdcall ColREST(PGLOBAL g, PTOS tp, char *tab, char *db, bool info)
 /***********************************************************************/
 bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff)
 {
-  char    filename[_MAX_PATH + 1];
+	char    filename[_MAX_PATH + 1];
   int     rc = 0, n;
-  LPCSTR  ftype;
+	bool    xt = trace(515);
+	LPCSTR  ftype;
+	XGETREST grf = GetRestFunction(g);
+
+	if (!grf)
+		return true;
 
 #if defined(MARIADB)
   ftype = GetStringCatInfo(g, "Type", "JSON");
@@ -135,7 +216,7 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff)
   ftype = GetStringCatInfo(g, "Ftype", "JSON");
 #endif  // !MARIADB
 
-  if (trace(515))
+  if (xt)
     htrc("ftype = %s am = %s\n", ftype, SVP(am));
 
   n = (!stricmp(ftype, "JSON")) ? 1
@@ -157,9 +238,9 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff)
   strncat(strcpy(filename, GetPath()), Fn, _MAX_PATH);
 
   // Retrieve the file from the web and copy it locally
-  rc = restGetFile(g, Http, Uri, filename);
+	rc = grf(g->Message, xt, Http, Uri, filename);
 
-  if (trace(515))
+  if (xt)
     htrc("Return from restGetFile: rc=%d\n", rc);
 
   if (rc)
@@ -175,7 +256,7 @@ bool RESTDEF::DefineAM(PGLOBAL g, LPCSTR am, int poff)
   if (Tdp && Tdp->Define(g, Cat, Name, Schema, "REST"))
     Tdp = NULL; // Error occured
 
-  if (trace(515))
+  if (xt)
     htrc("Tdp defined\n", rc);
 
   // Return true in case of error

storage/connect/tabutil.cpp

@@ -59,11 +59,23 @@ int GetConvSize(void);
 /*  Used by MYSQL tables to get MySQL parameters from the calling proxy */
 /*  table (PROXY, TBL, XCL, or OCCUR) when used by one of these.        */
 /************************************************************************/
-void Remove_tshp(PCATLG cat)
+TABLE_SHARE *Remove_tshp(PCATLG cat)
 {
-  ((MYCAT*)cat)->GetHandler()->tshp = NULL;
+  TABLE_SHARE *s = ((MYCAT*)cat)->GetHandler()->tshp;
+
+	((MYCAT*)cat)->GetHandler()->tshp = NULL;
+	return s;
 } // end of Remove_thsp
 
+/************************************************************************/
+/*  Used by MYSQL tables to get MySQL parameters from the calling proxy */
+/*  table (PROXY, TBL, XCL, or OCCUR) when used by one of these.        */
+/************************************************************************/
+void Restore_tshp(PCATLG cat, TABLE_SHARE *s)
+{
+	((MYCAT*)cat)->GetHandler()->tshp = s;
+} // end of Restore_thsp
+
 /************************************************************************/
 /*  GetTableShare: allocates and open a table share.                    */
 /************************************************************************/

storage/connect/tabutil.h

@@ -18,7 +18,8 @@ TABLE_SHARE *GetTableShare(PGLOBAL g, THD *thd, const char *db,
 PQRYRES TabColumns(PGLOBAL g, THD *thd, const char *db, 
                                         const char *name, bool& info);
 
-void Remove_tshp(PCATLG cat);
+TABLE_SHARE *Remove_tshp(PCATLG cat);
+void Restore_tshp(PCATLG cat, TABLE_SHARE *s);
 
 /* -------------------------- PROXY classes -------------------------- */
 

storage/connect/tabvct.cpp

@@ -115,11 +115,14 @@ bool VCTDEF::DefineAM(PGLOBAL g, LPCSTR, int poff)
 
   Recfm = RECFM_VCT;
 
+	// poff is no more in use; This will have to be revisited
+#if 0
   // For packed files the logical record length is calculated in poff
   if (poff != Lrecl) {
     Lrecl = poff;
     SetIntCatInfo("Lrecl", poff);
     } // endif poff
+#endif // 0
 
   Padded = false;
   Blksize = 0;

storage/connect/xindex.cpp

@@ -659,7 +659,7 @@ bool XINDEX::Make(PGLOBAL g, PIXDEF sxp)
   /*  Not true for DBF tables because of eventual soft deleted lines.  */
   /*  Note: for Num_K = 1 any non null value is Ok.                    */
   /*********************************************************************/
-  if (Srtd && !filp && Tdbp->Ftype != RECFM_VAR 
+  if (Srtd && !filp && Tdbp->Ftype != RECFM_VAR && Tdbp->Ftype != RECFM_CSV
                     && Tdbp->Txfp->GetAmType() != TYPE_AM_DBF) {
     Incr = (Num_K > 1) ? To_Rec[1] : Num_K;
     PlgDBfree(Record);
@@ -837,7 +837,8 @@ bool XINDEX::SaveIndex(PGLOBAL g, PIXDEF sxp)
     case RECFM_FIX: ftype = ".fnx"; break;
     case RECFM_BIN: ftype = ".bnx"; break;
     case RECFM_VCT: ftype = ".vnx"; break;
-    case RECFM_DBF: ftype = ".dbx"; break;
+		case RECFM_CSV: ftype = ".cnx"; break;
+		case RECFM_DBF: ftype = ".dbx"; break;
     default:
       sprintf(g->Message, MSG(INVALID_FTYPE), Tdbp->Ftype);
       return true;
@@ -990,7 +991,8 @@ bool XINDEX::Init(PGLOBAL g)
     case RECFM_FIX: ftype = ".fnx"; break;
     case RECFM_BIN: ftype = ".bnx"; break;
     case RECFM_VCT: ftype = ".vnx"; break;
-    case RECFM_DBF: ftype = ".dbx"; break;
+		case RECFM_CSV: ftype = ".cnx"; break;
+		case RECFM_DBF: ftype = ".dbx"; break;
     default:
       sprintf(g->Message, MSG(INVALID_FTYPE), Tdbp->Ftype);
       return true;
@@ -1243,7 +1245,8 @@ bool XINDEX::MapInit(PGLOBAL g)
     case RECFM_FIX: ftype = ".fnx"; break;
     case RECFM_BIN: ftype = ".bnx"; break;
     case RECFM_VCT: ftype = ".vnx"; break;
-    case RECFM_DBF: ftype = ".dbx"; break;
+		case RECFM_CSV: ftype = ".cnx"; break;
+		case RECFM_DBF: ftype = ".dbx"; break;
     default:
       sprintf(g->Message, MSG(INVALID_FTYPE), Tdbp->Ftype);
       return true;
@@ -1457,7 +1460,8 @@ bool XINDEX::GetAllSizes(PGLOBAL g,/* int &ndif,*/ int &numk)
     case RECFM_FIX: ftype = ".fnx"; break;
     case RECFM_BIN: ftype = ".bnx"; break;
     case RECFM_VCT: ftype = ".vnx"; break;
-    case RECFM_DBF: ftype = ".dbx"; break;
+		case RECFM_CSV: ftype = ".cnx"; break;
+		case RECFM_DBF: ftype = ".dbx"; break;
     default:
       sprintf(g->Message, MSG(INVALID_FTYPE), Tdbp->Ftype);
       return true;