View Ticket
Ticket Hash: bd6188edd4290bfdcff4a8ef9355f5280496eda5
Title: dir2c: Output data as character array and not as string
Status: Open Type: Feature Request
Severity: Minor Priority: Immediate
Subsystem: Tcl Resolution: Open
Last Modified: 2020-05-11 16:40:12
Version Found In: trunk
Description:
Dear Roy,

thank you for great tcl2c.tcl and its framework.
I am learning each day and digging miself through the mechanisms.
Sorry to be a novice.
And sorry to use Microsoft compilers.

The compiler MS-VC2015 (VC9) has a string constant size limit of 65535 characters.
A possibility to avoid this limit is to use an array of characters, which have a limit of size_z = 0x7CFFFFFF.

The patch below changes dir2c.tcl to output each file within one character array named  cvfs_data_$idx as follows:

<verbatim>
/* Data of boot.tcl */
static unsigned char cvfs_data_2[] = {
    0x70, 0x72, 0x6f, 0x63, 0x20, 0x74, 0x63, 0x6c, 0x49, 0x6e, 0x69, 0x74, 0x20, 0x7b, 0x7d, 0x20,
...
static struct cvfs_data cvfs_tcl_data[] = {
...
    {
        /* name  */ "boot.tcl",
        /* index */ 2,
        /* type  */ CVFS_FILETYPE_FILE,
        /* size  */ 2887,
        /* data  */ cvfs_data_2,
        /* free  */ 0,
    },
</verbatim>

The patch allows to use this for all files or files larger than the character length limit.

I may miss another disadvantage of this method. I suppose, it only avoids the implicite trailing zero byte.

Thank you,
Harald

Patch:
<verbatim>
--- C:/Users/oehhar/AppData/Local/Temp/dir2c.tcl-revBASE.svn001.tmp.tcl    Fri Apr 17 17:29:24 2020
+++ C:/oehhar/elmicron/projekte/el1005_scanlink_dll/source/c-vfs/dir2c.tcl    Mon Apr 20 17:30:25 2020
@@ -66,26 +66,44 @@
 proc stringify {data} {
     set ret "\""
     for {set idx 0} {$idx < [string length $data]} {incr idx} {
         binary scan [string index $data $idx] H* char
 
         append ret "\\x${char}"
 
-        if {($idx % 20) == 0 && $idx != 0} {
+        if {(($idx+1) % 20) == 0} {
             append ret "\"\n\""
         }
     }
 
     set ret [string trim $ret "\n\""]
 
     set ret "\"$ret\""
 
     return $ret
 }
 
+# Convert data to a character array
+proc character_array {data} {
+    set ret "\{\n\t"
+    for {set idx 0} {$idx < [string length $data]} {incr idx} {
+        binary scan [string index $data $idx] H* char
+        append ret "0x${char}, "
+
+        if {(($idx+1) % 16) == 0 } {
+            append ret "\n\t"
+        }
+    }
+    set ret [string trimright $ret "\n, \t"]
+
+    append ret "\n\}"
+
+    return $ret
+}
+
 # Encrypt the data
 proc random_byte {} {
     set value [expr {int(256 * rand())}]
 
     return $value
 }
 
@@ -303,64 +321,80 @@
 
     return(0);
 }
 
 #  endif /* !LOADED_CVFS_COMMON */}
 puts ""
 
-puts "static struct cvfs_data ${code_tag}_data\[\] = {"
-puts "\t{"
-puts "\t\t/* Index 0 cannot be used because we use the value 0 to represent failure */"
-puts "\t\t/* name  */ NULL,"
-puts "\t\t/* index */ 0,"
-puts "\t\t/* type  */ 0,"
-puts "\t\t/* size  */ 0,"
-puts "\t\t/* data  */ NULL,"
-puts "\t\t/* free  */ 0,"
-puts "\t},"
+# C struct data output buffer (must be delayed after definition of file data)
+set struct_data ""
+
 for {set idx 1} {$idx < [llength $files]} {incr idx} {
     set file [lindex $files $idx]
     set shortfile [shorten_file $startdir $file]
 
     unset -nocomplain finfo type
     file stat $file finfo
 
     switch -- $finfo(type) {
         "file" {
             set size $finfo(size)
 
             set fd [open $file]
             fconfigure $fd -translation binary
-            set data [read $fd]
+            set data_in [read $fd]
             close $fd
 
             if {$obsfucate} {
                 set type "CVFS_FILETYPE_ENCRYPTED_FILE"
-                set data "(unsigned char *) [stringify [encrypt $data $obsfucation_key]]"
+                set data_in [encrypt $data $obsfucation_key]
             } else {
                 set type "CVFS_FILETYPE_FILE"
-                set data "(unsigned char *) [stringify $data]"
             }
+            # Microsoft VC9 compiler has a string limit of 65535 bytes
+            # so put all larger files into an array which has a higher
+            # string limit
+            #if {$size <= 65535} {
+            #    set data "(unsigned char *) [stringify $data_in]"
+            #} else {
+                puts "/* Data of $shortfile */"
+                set variable_name cvfs_data_$idx
+                set data $variable_name
+                puts "static unsigned char ${variable_name}\[\] =\
+                        [character_array $data_in];"
+            #}
         }
         "directory" {
             set type "CVFS_FILETYPE_DIR"
             set data "NULL"
             set size 0
         }
     }
 
-    puts "\t{"
-    puts "\t\t/* name  */ \"$shortfile\","
-    puts "\t\t/* index */ $idx,"
-    puts "\t\t/* type  */ $type,"
-    puts "\t\t/* size  */ $size,"
-    puts "\t\t/* data  */ $data,"
-    puts "\t\t/* free  */ 0,"
-    puts "\t},"
+    append struct_data "\t{\n"\
+            "\t\t/* name  */ \"$shortfile\",\n"\
+            "\t\t/* index */ $idx,\n"\
+            "\t\t/* type  */ $type,\n"\
+            "\t\t/* size  */ $size,\n"\
+            "\t\t/* data  */ $data,\n"\
+            "\t\t/* free  */ 0,\n"\
+            "\t},\n"
 }
+puts ""
+puts "static struct cvfs_data ${code_tag}_data\[\] = {"
+puts "\t{"
+puts "\t\t/* Index 0 cannot be used because we use the value 0 to represent failure */"
+puts "\t\t/* name  */ NULL,"
+puts "\t\t/* index */ 0,"
+puts "\t\t/* type  */ 0,"
+puts "\t\t/* size  */ 0,"
+puts "\t\t/* data  */ NULL,"
+puts "\t\t/* free  */ 0,"
+puts "\t},"
+puts -nonewline $struct_data
 puts "};"
 puts ""
 
 puts "static unsigned long ${code_tag}_lookup_index(const char *path) {"
 puts "\tswitch (cvfs_hash((unsigned char *) path)) {"
 
 for {set idx 1} {$idx < [llength $files]} {incr idx} {

</verbatim>
User Comments:
anonymous (claiming to be oehhar) added on 2020-05-11 16:38:04:

Dear Roy,

sorry, this patch also has an issue. The obfuscated version does not work any more (wrong variable data instead data_in).

Here is a corrected total patch including ticket [127ac40147]


--- C:/oehhar/elmicron/projekte/el1005_scanlink_dll/source/c-vfs/dir2c.tcl	Mon May 11 07:31:09 2020
+++ C:/oehhar/elmicron/projekte/el1005_scanlink_dll/source/c-vfs/dir2c_ori.tcl	Wed Jan 22 20:53:52 2020
@@ -70,7 +70,7 @@
 
 		append ret "\\x${char}"
 
-		if {(($idx+1) % 20) == 0} {
+		if {($idx % 20) == 0 && $idx != 0} {
 			append ret "\"\n\""
 		}
 	}
@@ -82,24 +82,6 @@
 	return $ret
 }
 
-# Convert data to a character array
-proc character_array {data} {
-	set ret "\{\n\t"
-	for {set idx 0} {$idx < [string length $data]} {incr idx} {
-		binary scan [string index $data $idx] H* char
-		append ret "0x${char}, "
-
-		if {(($idx+1) % 16) == 0 } {
-			append ret "\n\t"
-		}
-	}
-	set ret [string trimright $ret "\n, \t"]
-
-	append ret "\n\}"
-
-	return $ret
-}
-
 # Encrypt the data
 proc random_byte {} {
 	set value [expr {int(256 * rand())}]
@@ -325,9 +307,16 @@
 #  endif /* !LOADED_CVFS_COMMON */}
 puts ""
 
-# C struct data output buffer (must be delayed after definition of file data)
-set struct_data ""
-
+puts "static struct cvfs_data ${code_tag}_data\[\] = {"
+puts "\t{"
+puts "\t\t/* Index 0 cannot be used because we use the value 0 to represent failure */"
+puts "\t\t.name  = NULL,"
+puts "\t\t.index = 0,"
+puts "\t\t.type  = 0,"
+puts "\t\t.size  = 0,"
+puts "\t\t.data  = NULL,"
+puts "\t\t.free  = 0,"
+puts "\t},"
 for {set idx 1} {$idx < [llength $files]} {incr idx} {
 	set file [lindex $files $idx]
 	set shortfile [shorten_file $startdir $file]
@@ -341,28 +330,15 @@
 
 			set fd [open $file]
 			fconfigure $fd -translation binary
-			set data_in [read $fd]
+			set data [read $fd]
 			close $fd
 
 			if {$obsfucate} {
 				set type "CVFS_FILETYPE_ENCRYPTED_FILE"
-				set data_in [encrypt $data_in $obsfucation_key]
+				set data "(unsigned char *) [stringify [encrypt $data $obsfucation_key]]"
 			} else {
 				set type "CVFS_FILETYPE_FILE"
-			}
-			# Microsoft VC9 compiler has a string limit of 65535 bytes
-			# so put all larger files into an array which has a higher
-			# string limit
-			# For instance, the extra variable is used for all files.
-			# this may be deactivated by removing "0 &" below
-			if { 0 & $size <= 65535} {
-				set data "(unsigned char *) [stringify $data_in]"
-			} else {
-				puts "/* Data of $shortfile */"
-				set variable_name cvfs_data_$idx
-				set data $variable_name
-				puts "static unsigned char ${variable_name}\[\] =\
-						[character_array $data_in];"
+				set data "(unsigned char *) [stringify $data]"
 			}
 		}
 		"directory" {
@@ -372,27 +348,15 @@
 		}
 	}
 
-	append struct_data "\t{\n"\
-			"\t\t/* name  */ \"$shortfile\",\n"\
-			"\t\t/* index */ $idx,\n"\
-			"\t\t/* size  */ $size,\n"\
-			"\t\t/* type  */ $type,\n"\
-			"\t\t/* data  */ $data,\n"\
-			"\t\t/* free  */ 0,\n"\
-			"\t},\n"
+	puts "\t{"
+	puts "\t\t.name  = \"$shortfile\","
+	puts "\t\t.index = $idx,"
+	puts "\t\t.type  = $type,"
+	puts "\t\t.size  = $size,"
+	puts "\t\t.data  = $data,"
+	puts "\t\t.free  = 0,"
+	puts "\t},"
 }
-puts ""
-puts "static struct cvfs_data ${code_tag}_data\[\] = {"
-puts "\t{"
-puts "\t\t/* Index 0 cannot be used because we use the value 0 to represent failure */"
-puts "\t\t/* name  */ NULL,"
-puts "\t\t/* index */ 0,"
-puts "\t\t/* type  */ 0,"
-puts "\t\t/* size  */ 0,"
-puts "\t\t/* data  */ NULL,"
-puts "\t\t/* free  */ 0,"
-puts "\t},"
-puts -nonewline $struct_data
 puts "};"
 puts ""


anonymous (claiming to be oehhar) added on 2020-05-11 16:40:12:

I would love to be able to edit my own bad stuff.

The diff was in the wrong direction. Here is the corrected diff:


--- C:/oehhar/elmicron/projekte/el1005_scanlink_dll/source/c-vfs/dir2c_ori.tcl	Wed Jan 22 20:53:52 2020
+++ C:/oehhar/elmicron/projekte/el1005_scanlink_dll/source/c-vfs/dir2c.tcl	Mon May 11 07:31:09 2020
@@ -70,7 +70,7 @@
 
 		append ret "\\x${char}"
 
-		if {($idx % 20) == 0 && $idx != 0} {
+		if {(($idx+1) % 20) == 0} {
 			append ret "\"\n\""
 		}
 	}
@@ -82,6 +82,24 @@
 	return $ret
 }
 
+# Convert data to a character array
+proc character_array {data} {
+	set ret "\{\n\t"
+	for {set idx 0} {$idx < [string length $data]} {incr idx} {
+		binary scan [string index $data $idx] H* char
+		append ret "0x${char}, "
+
+		if {(($idx+1) % 16) == 0 } {
+			append ret "\n\t"
+		}
+	}
+	set ret [string trimright $ret "\n, \t"]
+
+	append ret "\n\}"
+
+	return $ret
+}
+
 # Encrypt the data
 proc random_byte {} {
 	set value [expr {int(256 * rand())}]
@@ -307,16 +325,9 @@
 #  endif /* !LOADED_CVFS_COMMON */}
 puts ""
 
-puts "static struct cvfs_data ${code_tag}_data\[\] = {"
-puts "\t{"
-puts "\t\t/* Index 0 cannot be used because we use the value 0 to represent failure */"
-puts "\t\t.name  = NULL,"
-puts "\t\t.index = 0,"
-puts "\t\t.type  = 0,"
-puts "\t\t.size  = 0,"
-puts "\t\t.data  = NULL,"
-puts "\t\t.free  = 0,"
-puts "\t},"
+# C struct data output buffer (must be delayed after definition of file data)
+set struct_data ""
+
 for {set idx 1} {$idx < [llength $files]} {incr idx} {
 	set file [lindex $files $idx]
 	set shortfile [shorten_file $startdir $file]
@@ -330,15 +341,28 @@
 
 			set fd [open $file]
 			fconfigure $fd -translation binary
-			set data [read $fd]
+			set data_in [read $fd]
 			close $fd
 
 			if {$obsfucate} {
 				set type "CVFS_FILETYPE_ENCRYPTED_FILE"
-				set data "(unsigned char *) [stringify [encrypt $data $obsfucation_key]]"
+				set data_in [encrypt $data_in $obsfucation_key]
 			} else {
 				set type "CVFS_FILETYPE_FILE"
-				set data "(unsigned char *) [stringify $data]"
+			}
+			# Microsoft VC9 compiler has a string limit of 65535 bytes
+			# so put all larger files into an array which has a higher
+			# string limit
+			# For instance, the extra variable is used for all files.
+			# this may be deactivated by removing "0 &" below
+			if { 0 & $size <= 65535} {
+				set data "(unsigned char *) [stringify $data_in]"
+			} else {
+				puts "/* Data of $shortfile */"
+				set variable_name cvfs_data_$idx
+				set data $variable_name
+				puts "static unsigned char ${variable_name}\[\] =\
+						[character_array $data_in];"
 			}
 		}
 		"directory" {
@@ -348,15 +372,27 @@
 		}
 	}
 
-	puts "\t{"
-	puts "\t\t.name  = \"$shortfile\","
-	puts "\t\t.index = $idx,"
-	puts "\t\t.type  = $type,"
-	puts "\t\t.size  = $size,"
-	puts "\t\t.data  = $data,"
-	puts "\t\t.free  = 0,"
-	puts "\t},"
+	append struct_data "\t{\n"\
+			"\t\t/* name  */ \"$shortfile\",\n"\
+			"\t\t/* index */ $idx,\n"\
+			"\t\t/* size  */ $size,\n"\
+			"\t\t/* type  */ $type,\n"\
+			"\t\t/* data  */ $data,\n"\
+			"\t\t/* free  */ 0,\n"\
+			"\t},\n"
 }
+puts ""
+puts "static struct cvfs_data ${code_tag}_data\[\] = {"
+puts "\t{"
+puts "\t\t/* Index 0 cannot be used because we use the value 0 to represent failure */"
+puts "\t\t/* name  */ NULL,"
+puts "\t\t/* index */ 0,"
+puts "\t\t/* type  */ 0,"
+puts "\t\t/* size  */ 0,"
+puts "\t\t/* data  */ NULL,"
+puts "\t\t/* free  */ 0,"
+puts "\t},"
+puts -nonewline $struct_data
 puts "};"
 puts ""