Add SQL template options

Author: ibmmqmet

CHANGES.md

@@ -14,14 +14,14 @@ June 2016 (v1.1.1)
 * Print the buildtime to help know what version is in use
 * Clearer printing of which records were unrecognised
 
-June 2016 (v1.1.2)
+June 2016 (v1.2)
 * Add SQL mode to generate DDL corresponding to column headings
 * printWQ now has more descriptive column names
 
 July 2016
 * Add progress indicator
 
-August 2016
+August 2016 (v1.2.1)
 * Add correlator field to better link WQ/WTID/WTAS
 * Add user-written content in docs directory. How to use SQL.
 
@@ -34,39 +34,42 @@ February 2017
 * STCK durations printed as just microseconds value when in SQL mode. Still
 split as seconds/microseconds otherwise.
 
-May 2017
+May 2017 (v2.0.0)
 * Added support for the 903 SMF extensions
 * Using a VERS flag during the build to select correct base header file
 and features
 
-Oct 2017
+Oct 2017 (v2.0.1)
 * Found that some 115 subtypes do not have a QWHS structure correctly
 created; cannot rely on it being there
 
-Jan 2018
+Jan 2018 (v3.0.0)
 * QCTDNS structure is not initialised when there has been no activity
 * Can now process files that have not been downloaded with the RDW option,
 as some sites do not have suitable ftp package available. New option
 is "-f NORDW".
 
-Feb 2018
+Feb 2018 (v3.1.0)
 * Added a simple "checkpoint" manager to recover after failures of the
 formatter. This may help when you run out of disk space while formatting
 very large output - the program may be able to restart from a more recent
 checkpoint.
 * Discovered a few fields in the WTAS structure are not used in the way
 the documentation and product header files claim.
 * Some SMF integers are too large for the INTEGER datatype. The unsigned
-32-bits compared to the SQL which is signed 32-bit, So switching
+32-bits compared to the SQL which is signed 32-bit. So switching
 to always use BIGINT.
 
-Mar 2018
+Mar 2018 (v3.2.0)
 * Add code to help add INDEX to columns in the SQL DDL output
 * Add optimisation to Makefiles
 * Even 64-bit values may be too large when they are not really counters
 
-Apr 2018
+Apr 2018 (v4.0)
 * Add -f option to generate JSON output format
 * Allow -o to specify stdout for output. Modify printf to select stdout/stderr
 as appropriate
-* Add processing rate (MB/sec) to output
+* Add data processing rate (MB/sec) to output
+
+May 2018 (v4.1.0)
+* Ability to add SQL/DDL commands to generated DDL file from templates.

bin/aix/convH

@@ -117,6 +117,8 @@ BOOL  printHeaders = TRUE;
 BOOL  useRDW = TRUE;
 BOOL  streamOutput = FALSE;
 BOOL  streamInput = TRUE;
+char *ddlTemplateOpen = NULL;
+char *ddlTemplateClose = NULL;
 commonFields_t commonF = {0};
 enum outputFormat_e outputFormat = OF_CSV;
 FILE *infoStream;
@@ -239,7 +241,7 @@ int main( int argc, char *argv[] )
   /******************************************************************/
   /* Parse command-line parameters                                  */
   /******************************************************************/
-  while((c = mqgetopt(argc, argv, "acd:f:h:i:m:o:rst:")) != EOF)
+  while((c = mqgetopt(argc, argv, "acd:e:f:h:i:m:o:p:rst:")) != EOF)
   {
     switch(c)
     {
@@ -253,6 +255,9 @@ int main( int argc, char *argv[] )
       case 'd':
         debugLevel = atoi(mqoptarg);
         break;
+      case 'e':
+         ddlTemplateClose = mqoptarg;
+         break;
       case 'f':
         for (i=0;i<strlen(mqoptarg);i++)
           mqoptarg[i] = toupper(mqoptarg[i]);
@@ -288,6 +293,9 @@ int main( int argc, char *argv[] )
           streamOutput=TRUE;
         }
         break;
+      case 'p': /* 'p' for Pool ... "create bufferpool" is default cmd */
+        ddlTemplateOpen = mqoptarg;
+        break;
       case 'r':
         addEquals = 0;
         break;
@@ -1002,6 +1010,10 @@ int main( int argc, char *argv[] )
   remove(checkPointFileName);
   pos = ftello(fp);
 
+  if (outputFormat == OF_SQL && ddlTemplateClose != NULL) {
+       closeFinalDDL();
+  }
+
 mod_exit:
 
   fflush(NULL); /* Ensure all streams flushed if possible */
@@ -1296,6 +1308,8 @@ static void Usage(void)
   fprintf(infoStream,"Usage: mqsmfcsv [-o <output dir>] [-a] [ -d <level> ]\n");
   fprintf(infoStream,"         [-h yes|no] [ -i <input file> [-m <max records>]\n");
   fprintf(infoStream,"         [-f RDW | NORDW | JSON | SQL | CSV ] \n");
+  fprintf(infoStream,"         [-p <template DDL file prefix>  ] \n");
+  fprintf(infoStream,"         [-e <template DDL file ending>  ] \n");
   fprintf(infoStream,"         [-r] [-c] [-t <ticker>]\n");
   fprintf(infoStream,"  -a               Append to files if they exist. Default is overwrite.\n");
   fprintf(infoStream,"  -c               Recover after aborted run by using the checkpoint.\n");
@@ -1306,6 +1320,9 @@ static void Usage(void)
   fprintf(infoStream,"  -m <Max records> End after formatting M records. Default to process all.\n");
   fprintf(infoStream,"  -o <Directory>   Where to put output files. Default to current directory.\n");
   fprintf(infoStream,"                   Use '-' to send output to stdout.\n");
+  fprintf(infoStream,"  -p <file>        Add contents of <file> to start of generated DDL.\n");
+  fprintf(infoStream,"                   Set to '-' to bypass default commands.\n");
+  fprintf(infoStream,"  -e <file>        Add contents of <file> to end of generated DDL.\n");
   fprintf(infoStream,"  -r               Do not print '=' on numeric-looking strings.\n");
   fprintf(infoStream,"  -s               (Deprecated) SQL mode - generate DDL for tables.\n");
   fprintf(infoStream,"  -t <Ticker>      Print progress message every T records.\n");

bin/aix/mqsmfcsv

@@ -161,6 +161,7 @@ extern void printWTID  (wtid *, unsigned char *);
 
 extern void openDDL    (char *);
 extern void closeDDL   (char *);
+extern void closeFinalDDL   (void);
 extern void printDDL   (char *,int, int);
 extern char *formatDDL (char *);
 extern char *formatDDLMaxLength (char *,int);
@@ -436,6 +437,8 @@ extern unsigned short recordSubType;
 extern commonFields_t commonF;
 extern char tmpHead[];
 extern FILE *infoStream;
+extern char *ddlTemplateOpen;
+extern char *ddlTemplateClose;
 
 enum outputFormat_e { OF_CSV=0, OF_SQL=1, OF_JSON=2 };
 extern enum outputFormat_e outputFormat;

bin/linux/mqsmfcsv

@@ -13,6 +13,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <ctype.h>
+#include <errno.h>
 #include "mqsmf.h"
 
 static FILE *fp = NULL;
@@ -23,6 +24,9 @@ char *schema = "MQSMF";
 char *indexSet[INDEXSETSIZE] = {0};
 int   nextIndex = 0;
 
+char templateLine[1024];
+FILE *templateCloseFp = NULL;
+
 /*
  * The DDL generated here is very simple. The
  * table is created with a fixed name and relevant
@@ -44,10 +48,51 @@ void openDDL(char *name)
 
    if (!fp)
    {
+     char *c = NULL;
      fp = fopenext("MQTABLES","ddl", &newFile);
      if (!fp) /* fopenext will print detailed error message */
        exit(1);
+
+     /******************************************************************/
+     /* Add some template DDL commands to the output. By default some  */
+     /* commands to create large pagesize items are written. If the    */
+     /* specified template file is called "-" then nothing is written. */
+     /* Otherwise copy from the template to the real DDL               */
+     /******************************************************************/
+     if (ddlTemplateOpen) {
+       if (strcmp(ddlTemplateOpen,"-")) {
+         FILE *templateOpenFp;
+         templateOpenFp = fopen(ddlTemplateOpen,"r");
+         if (!templateOpenFp) {
+          fprintf(stderr," Cannot open file %s. Error \"%s\" (%d) \n",
+            ddlTemplateOpen, strerror(errno),errno);
+            exit(1);
+         }
+         do {
+           c = fgets(templateLine,sizeof(templateLine)-1,templateOpenFp);
+           if (c)
+             fprintf(fp,"%s",templateLine);
+         } while(c);
+         fprintf(fp,"\n");
+       }
+     } else {
+       /* These are some default commands */
+       fprintf(fp,"CREATE BUFFERPOOL MQP8K IMMEDIATE ALL DBPARTITIONNUMS SIZE 1000 AUTOMATIC PAGESIZE 8192;\n");
+       fprintf(fp,"CREATE TABLESPACE MQP8K PAGESIZE 8192 BUFFERPOOL MQP8K;\n");
+       fprintf(fp,"\n");
+
+     }
+  }
+
+  if (ddlTemplateClose && !templateCloseFp) {
+    templateCloseFp = fopen(ddlTemplateClose,"r");
+    if (!templateCloseFp) {
+      fprintf(stderr, " Cannot open file %s. Error \"%s\" (%d) \n",
+         ddlTemplateClose,strerror(errno),errno);
+    }
+
    }
+
    fprintf(fp,"DROP   TABLE %s.%s;  \n",schema,name);
 
    /* If you want more sophisticated SQL, then you could change this */
@@ -75,6 +120,28 @@ void closeDDL(char *name)
   return;
 }
 
+void closeFinalDDL(void) {
+
+  char *c;
+  if (!templateCloseFp)
+    return;
+
+  do {
+    c = fgets(templateLine,sizeof(templateLine)-1,templateCloseFp);
+    if (c)
+       fprintf(fp,"%s",templateLine);
+  } while(c);
+  fprintf(fp,"\n");
+
+  fclose(fp);
+  fclose(templateCloseFp);
+
+  fp = NULL;
+  templateCloseFp = NULL;
+
+  return;
+}
+
 void addIndex(char *key)
 {
   if (nextIndex <INDEXSETSIZE) {