MDEV-31625 connect engine file_type=DBF insert fails

Add the end of file marker x1A to the DBF file and handle it
correctly to preserve interoperability with Libreoffice, and others
that have followed the DBF spec.

The file open mode of "a+" was problematic because Linux and the OSX,
the previous main development mode are inconsistent (see man fopen).
The main problem per the bug report was the inability to fseek back to the
beginning to update the records in the header.

As such the "a+" mode is remove and "w+b" is used inserting to a new file
and "r+b" is used for appending to the file.

In DBFFAM::CloseTableFile move PlugCloseFile down to close the file in
all modes.

The year unlike the comments is always since 1900. Use the
YYYY-MM-DD as an unabigious form during tracing.

Thanks for Mr. Zoltan Duna for the descriptive bug report.

storage/connect/filamdbf.cpp

@@ -335,9 +335,9 @@ PQRYRES DBFColumns(PGLOBAL g, PCSZ dp, PCSZ fn, PTOS topt, bool info)
           hp->Headlen(), hp->Reclen(), fields);
     htrc("flags(iem)=%d,%d,%d cp=%d\n", hp->Incompleteflag,
           hp->Encryptflag, hp->Mdxflag, hp->Language);
-    htrc("%hd records, last changed %02d/%02d/%d\n",
-          hp->Records(), hp->Filedate[1], hp->Filedate[2],
-          hp->Filedate[0] + (hp->Filedate[0] <= 30) ? 2000 : 1900);
+    htrc("%hd records, last changed %04d-%02d-%02d\n",
+          hp->Records(),
+          hp->Filedate[0] + 1900, hp->Filedate[1], hp->Filedate[2]);
     htrc("Field    Type  Offset  Len  Dec  Set  Mdx\n");
     } // endif trace
 
@@ -605,8 +605,7 @@ bool DBFFAM::OpenTableFile(PGLOBAL g)
       strcpy(opmode, (UseTemp) ? "rb" : "r+b");
       break;
     case MODE_INSERT:
-      // Must be in text mode to remove an eventual EOF character
-      strcpy(opmode, "a+");
+      strcpy(opmode, Records ? "r+b" : "w+b");
       break;
     default:
       snprintf(g->Message, sizeof(g->Message), MSG(BAD_OPEN_MODE), mode);
@@ -619,7 +618,7 @@ bool DBFFAM::OpenTableFile(PGLOBAL g)
   if (!(Stream = PlugOpenFile(g, filename, opmode))) {
     if (trace(1))
       htrc("%s\n", g->Message);
-    
+
     return (mode == MODE_READ && errno == ENOENT)
             ? PushWarning(g, Tdbp) : true;
     } // endif Stream
@@ -643,6 +642,7 @@ bool DBFFAM::AllocateBuffer(PGLOBAL g)
   {
   char c;
   int  rc;
+  int len;
   MODE mode = Tdbp->GetMode();
 
   Buflen = Blksize;
@@ -664,7 +664,7 @@ bool DBFFAM::AllocateBuffer(PGLOBAL g)
     /************************************************************************/
     /*  If this is a new file, the header must be generated.                */
     /************************************************************************/
-    int len = GetFileLength(g);
+    len = GetFileLength(g);
 
     if (!len) {
       // Make the header for this DBF table file
@@ -702,7 +702,7 @@ bool DBFFAM::AllocateBuffer(PGLOBAL g)
       header->Version = DBFTYPE;
       t = time(NULL) - (time_t)DTVAL::GetShift();
       datm = gmtime(&t);
-      header->Filedate[0] = datm->tm_year - 100;
+      header->Filedate[0] = datm->tm_year;
       header->Filedate[1] = datm->tm_mon + 1;
       header->Filedate[2] = datm->tm_mday;
       header->SetHeadlen((ushort)hlen);
@@ -793,8 +793,12 @@ bool DBFFAM::AllocateBuffer(PGLOBAL g)
   /**************************************************************************/
   /*  Position the file at the begining of the data.                        */
   /**************************************************************************/
-  if (Tdbp->GetMode() == MODE_INSERT)
-    rc = fseek(Stream, 0, SEEK_END);
+  if (Tdbp->GetMode() == MODE_INSERT) {
+    if (len)
+      rc = fseek(Stream, -1, SEEK_END);
+    else
+      rc = fseek(Stream, 0, SEEK_END);
+    }
   else
     rc = fseek(Stream, Headlen, SEEK_SET);
 
@@ -979,6 +983,7 @@ void DBFFAM::CloseTableFile(PGLOBAL g, bool abort)
     Rbuf = CurNum--;
 //  Closing = true;
     wrc = WriteBuffer(g);
+    fputc(0x1a, Stream);
   } else if (mode == MODE_UPDATE || mode == MODE_DELETE) {
     if (Modif && !Closing) {
       // Last updated block remains to be written
@@ -1003,35 +1008,27 @@ void DBFFAM::CloseTableFile(PGLOBAL g, bool abort)
   } // endif's mode
 
   if (Tdbp->GetMode() == MODE_INSERT) {
-    int n = ftell(Stream) - Headlen;
-
-    rc = PlugCloseFile(g, To_Fb);
+    int n = ftell(Stream) - Headlen - 1;
 
     if (n >= 0 && !(n % Lrecl)) {
       n /= Lrecl;                       // New number of lines
 
       if (n > Records) {
         // Update the number of rows in the file header
-        char filename[_MAX_PATH];
-
-        PlugSetPath(filename, To_File, Tdbp->GetPath());
-        if ((Stream= global_fopen(g, MSGID_OPEN_MODE_STRERROR, filename, "r+b")))
-        {
-          char nRecords[4];
-          int4store(nRecords, n);
-
-          fseek(Stream, 4, SEEK_SET);     // Get header.Records position
-          fwrite(nRecords, sizeof(nRecords), 1, Stream);
-          fclose(Stream);
-          Stream= NULL;
-          Records= n;                    // Update Records value
-        }
+        char nRecords[4];
+        int4store(nRecords, n);
+
+        fseek(Stream, 4, SEEK_SET);     // Get header.Records position
+        fwrite(nRecords, sizeof(nRecords), 1, Stream);
+        Stream= NULL;
+        Records= n;                    // Update Records value
       } // endif n
 
     } // endif n
 
-  } else  // Finally close the file
-    rc = PlugCloseFile(g, To_Fb);
+  }
+  // Finally close the file
+  rc = PlugCloseFile(g, To_Fb);
 
  fin:
   if (trace(1))

storage/connect/mysql-test/connect/r/dbf.result

@@ -64,6 +64,24 @@ t1	CREATE TABLE `t1` (
   `a` int(11) NOT NULL
 ) ENGINE=CONNECT DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci `TABLE_TYPE`=DBF `FILE_NAME`='t1.dbf'
 INSERT INTO t1 VALUES (10),(20);
+CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf');
+--------	--------
+FileSize	91
+DBF_Version	03
+NRecords	2
+FirstRecPos	66
+RecLength	12
+TableFlags	0000
+CodePageMark	00
+---	---
+FieldN	0
+Name	a
+Type	N
+Offset	0
+Length	11
+Dec	0
+Flags	00
+--------	--------
 SELECT * FROM t1;
 a
 10
@@ -89,6 +107,24 @@ t1	CREATE TABLE `t1` (
   `a` int(11) NOT NULL
 ) ENGINE=CONNECT DEFAULT CHARSET=latin1 COLLATE=latin1_swedish_ci `TABLE_TYPE`=DBF `FILE_NAME`='t1.dbf' `READONLY`=NO
 INSERT INTO t1 VALUES (30);
+CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf');
+--------	--------
+FileSize	103
+DBF_Version	03
+NRecords	3
+FirstRecPos	66
+RecLength	12
+TableFlags	0000
+CodePageMark	00
+---	---
+FieldN	0
+Name	a
+Type	N
+Offset	0
+Length	11
+Dec	0
+Flags	00
+--------	--------
 SELECT * FROM t1;
 a
 10
@@ -137,7 +173,7 @@ a
 test
 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf');
 --------	--------
-FileSize	77
+FileSize	78
 DBF_Version	03
 NRecords	1
 FirstRecPos	66
@@ -171,7 +207,7 @@ a	b	c
 2	2	2
 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf');
 --------	--------
-FileSize	194
+FileSize	195
 DBF_Version	03
 NRecords	2
 FirstRecPos	130
@@ -264,7 +300,7 @@ a
 -9223372036854775808
 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf');
 --------	--------
-FileSize	108
+FileSize	109
 DBF_Version	03
 NRecords	2
 FirstRecPos	66
@@ -308,7 +344,7 @@ a
 -32768
 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf');
 --------	--------
-FileSize	80
+FileSize	81
 DBF_Version	03
 NRecords	2
 FirstRecPos	66
@@ -338,7 +374,7 @@ LENGTH(a)
 255
 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf');
 --------	--------
-FileSize	322
+FileSize	323
 DBF_Version	03
 NRecords	1
 FirstRecPos	66
@@ -419,7 +455,7 @@ a
 2001-01-01
 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf');
 --------	--------
-FileSize	75
+FileSize	76
 DBF_Version	03
 NRecords	1
 FirstRecPos	66
@@ -449,7 +485,7 @@ a
 123.0000
 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf');
 --------	--------
-FileSize	79
+FileSize	80
 DBF_Version	03
 NRecords	1
 FirstRecPos	66
@@ -481,7 +517,7 @@ a
 123456789.12345
 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf');
 --------	--------
-FileSize	108
+FileSize	109
 DBF_Version	03
 NRecords	2
 FirstRecPos	66
@@ -511,7 +547,7 @@ a
 10
 CALL dbf_header('MYSQLD_DATADIR/test/t1c.dbf');
 --------	--------
-FileSize	77
+FileSize	78
 DBF_Version	03
 NRecords	1
 FirstRecPos	66
@@ -538,7 +574,7 @@ a
 10
 CALL dbf_header('MYSQLD_DATADIR/test/t1c.dbf');
 --------	--------
-FileSize	77
+FileSize	78
 DBF_Version	03
 NRecords	1
 FirstRecPos	66
@@ -567,7 +603,7 @@ a
 10
 CALL dbf_header('MYSQLD_DATADIR/test/t1c.dbf');
 --------	--------
-FileSize	77
+FileSize	78
 DBF_Version	03
 NRecords	1
 FirstRecPos	66
@@ -604,7 +640,7 @@ c1	c2	i1	i2
 30	def	30	123
 CALL dbf_header('MYSQLD_DATADIR/test/t1.dbf');
 --------	--------
-FileSize	291
+FileSize	292
 DBF_Version	03
 NRecords	3
 FirstRecPos	162

storage/connect/mysql-test/connect/t/dbf.test

@@ -63,6 +63,11 @@ DELIMITER ;//
 CREATE TABLE t1 (a INT NOT NULL) ENGINE=CONNECT TABLE_TYPE=DBF FILE_NAME='t1.dbf';
 SHOW CREATE TABLE t1;
 INSERT INTO t1 VALUES (10),(20);
+--chmod 0777 $MYSQLD_DATADIR/test/t1.dbf
+--vertical_results
+--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR
+eval CALL dbf_header('$MYSQLD_DATADIR/test/t1.dbf');
+--horizontal_results
 SELECT * FROM t1;
 ALTER TABLE t1 READONLY=Yes;
 SHOW CREATE TABLE t1;
@@ -77,6 +82,11 @@ TRUNCATE TABLE t1;
 ALTER TABLE t1 READONLY=NO;
 SHOW CREATE TABLE t1;
 INSERT INTO t1 VALUES (30);
+--chmod 0777 $MYSQLD_DATADIR/test/t1.dbf
+--vertical_results
+--replace_result $MYSQLD_DATADIR MYSQLD_DATADIR
+eval CALL dbf_header('$MYSQLD_DATADIR/test/t1.dbf');
+--horizontal_results
 SELECT * FROM t1;
 DROP TABLE t1;
 --remove_file $MYSQLD_DATADIR/test/t1.dbf