updated logging

.gitignore

@@ -5,4 +5,8 @@
 json2excel
 
 # Ignore test data
-*.json
+*.json
+
+# Ignore Mac DS_Store files
+.DS_Store
+**/.DS_Store

README.md

@@ -0,0 +1,27 @@
+# json2excel
+
+This is a basic utility to take json and convert it into an OpenXml format xlsx file, compatible with Excel.
+
+It expects that the json input is formatted as an object containing an array of objects. Column names are generated based on the keys in the first nested object.
+
+## Command line options
+| Parameter     | Default Value | Description               |
+|---------------|---------------|---------------------------|
+| inputJson | input.json | File path of the input json to process |
+| outputFilename | output.xlsx | File path of the output file |
+| worksheetName | Sheet1 | Set the name of the worksheet, will be truncated to 31 characters |
+| boldTopRow | true | Sets the top row of the worksheet to bold |
+| freezeTopRow | true | Freezes the first row of the Excel worksheet |
+| autofilter | true | Sets the auto filter on the first row |
+| autowidth | true | Automatically set the column width to fit contents |
+| overwriteFile | false | Overwrite any existing output file instead of modifying in-place |
+
+## Advanced configuration
+
+Advanced settings can be provided via a top level json key named "config". Here is a table of options that can be set via this config key.
+
+| Key     | Example Value | Description               |
+|---------------|---------------|---------------------------|
+| key-order | "Column3,Column1,Column2"| Comma separated list of column names in the desired order |
+| overwrite-file | true | Boolean indicating whether output file should be overwritten if it already exists |
+| parent-node | "results" | Specify an alternate starting key for the spreadsheet data than just the first non-config key. Useful with json structures with multiple top-level keys |

cmd/main/main.go

@@ -6,6 +6,10 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"reflect"
+	"strings"
+	"time"
+	"unicode/utf8"
 
 	"github.com/iancoleman/orderedmap"
 	"github.com/xuri/excelize/v2"
@@ -13,9 +17,17 @@ import (
 
 // Initial concept from https://stackoverflow.com/q/68621039
 
+type Config struct {
+	keyOrder       map[int]string
+	parentOverride string
+}
+
+var config Config
+
 func main() {
 	//jsonFile := "test.json"
 	parentNode := "input"
+	configNode := "config"
 	//worksheetName := "Sheet2"
 	//outputFilename := "test.xlsx"
 
@@ -26,6 +38,8 @@ func main() {
 	var boldTopRow bool
 	var freezeTopRow bool
 	var autoFilter bool
+	var autoWidth bool
+	var overwriteFile bool
 
 	// Process command line arguments
 	flag.StringVar(&inputJson, "inputJson", "./input.json", "Full path to input json data file")
@@ -34,62 +48,21 @@ func main() {
 	flag.BoolVar(&boldTopRow, "bold-toprow", true, "Sets the top row of the worksheet to bold")
 	flag.BoolVar(&freezeTopRow, "freeze-toprow", true, "Freezes the first row of the Excel worksheet")
 	flag.BoolVar(&autoFilter, "autofilter", true, "Sets the auto filter on the first row")
+	flag.BoolVar(&autoWidth, "autowidth", true, "Automatically set the column width to fit contents")
+	flag.BoolVar(&overwriteFile, "overwrite", false, "Set to true to overwrite existing file rather than modifying in place")
 	flag.Parse()
 
 	var xlsx *excelize.File
 	var s []byte
 
-	var sheetIndex int
+	//var sheetIndex int
 	var err error
 
 	var cell string
 	var row, column int
 
-	// TODO - truncate worksheetName to the maximum 31 characters
-
-	// Check if xlsx file exists already, and if it does then open and append data
-	if fileExists(outputFilename) {
-		fmt.Printf("Output spreadsheet '%s' already exists.\n", outputFilename)
-		xlsx, err = excelize.OpenFile(outputFilename)
-		if err != nil {
-			fmt.Println(err)
-			os.Exit(1)
-		}
-
-		// Since we have an existing workbook, check if the sheet we want to write to already exists
-		sheetFound := false
-		for index, name := range xlsx.GetSheetMap() {
-			if name == worksheetName {
-				fmt.Printf("Found worksheet '%s' at index '%d'\n", worksheetName, index)
-				sheetFound = true
-			}
-		}
-		if !sheetFound {
-			// Create the sheet
-			fmt.Printf("Creating worksheet '%s'\n", worksheetName)
-			sheetIndex, err = xlsx.NewSheet(worksheetName)
-			if err != nil {
-				fmt.Printf("Error creating worksheet '%s' : %s\n", worksheetName, err)
-			}
-			// Set active worksheet
-			fmt.Printf("Setting active sheet to index %d", sheetIndex)
-			xlsx.SetActiveSheet(sheetIndex)
-		}
-
-	} else {
-		fmt.Printf("Creating output spreadsheet '%s'\n", outputFilename)
-		// Create the file
-		xlsx = excelize.NewFile()
-
-		// Rename the default Sheet1 to this worksheet name
-		if worksheetName != "Sheet1" {
-			fmt.Printf("Renaming default worksheet to '%s'\n", worksheetName)
-			err = xlsx.SetSheetName("Sheet1", worksheetName)
-			if err != nil {
-				fmt.Printf("Error setting sheet name to '%s': %s\n", worksheetName, err)
-			}
-		}
-	}
+	// Truncate worksheetName to the maximum 31 characters
+	worksheetName = TruncateString(worksheetName, 31)
 
 	// Read the json input file
 	if fileExists(inputJson) {
@@ -106,54 +79,131 @@ func main() {
 	o := orderedmap.New()
 	err = json.Unmarshal([]byte(s), &o)
 	if err != nil {
-		fmt.Printf("JSON Unmarshal error %s\n", err)
+		error := fmt.Sprintf("JSON Unmarshal error %s\n", err)
+		panic(error)
 	}
 
 	// Assume that our content is within the first top-level key
 	topLevel := o.Keys()
-	fmt.Printf("Detected toplevel json key as: '%s'\n", topLevel[0])
-	parentNode = topLevel[0]
+	fmt.Printf("Found %d top-level keys in json data\n", len(topLevel))
+	for i, key := range topLevel {
+		fmt.Printf("[%d] : %s\n", i, key)
+	}
+
+	// Check for config embedded in json
+	if strings.EqualFold(topLevel[0], configNode) && len(topLevel) > 1 {
+		fmt.Printf("Found configNode as toplevel json key, setting parentNode as '%s'\n", topLevel[1])
+		parentNode = topLevel[1]
+
+		config.keyOrder = make(map[int]string)
+
+		// Get a reference to the top level node we specified earlier
+		configInterface, ok := o.Get(configNode)
+		if !ok {
+			fmt.Printf("Missing key for multitype array when reading embedded config")
+		}
+
+		// Get an interface that we can work with to access the sub elements
+		// This doesn't seem necessary for some reason - maybe because there's only one level of depth to the configNode
+		//configSlice := configInterface
+		//fmt.Printf("%v\n", configSlice)
+
+		// Get the keys for the first element so we know what config options have been specified
+		configMap := configInterface.(orderedmap.OrderedMap)
+		configKeys := configMap.Keys()
+
+		// Parse each key into our config struct
+		for _, key := range configKeys {
+			if strings.EqualFold(key, "key-order") {
+				fmt.Printf("Found config element for key-order\n")
+				e, _ := configMap.Get(key)
+				for i, e := range strings.Split(e.(string), ",") {
+					config.keyOrder[i] = e
+				}
+				fmt.Printf("Column order is now : '%v'\n", config.keyOrder)
+			} else if strings.EqualFold(key, "overwrite-file") {
+				fmt.Printf("Found config element for overwriting output file\n")
+				e, _ := configMap.Get(key)
+				overwriteFile = e.(bool)
+			} else if strings.EqualFold(key, "parent-node") {
+				fmt.Printf("Found config element for forcing parent key for spreadsheet data\n")
+				e, _ := configMap.Get(key)
+				config.parentOverride = e.(string)
+			}
+		}
+	} else if strings.EqualFold(topLevel[0], configNode) {
+		error := "Only found config in first level of json keys"
+		panic(error)
+	} else {
+		fmt.Printf("Detected toplevel json key as: '%s'\n", topLevel[0])
+		parentNode = topLevel[0]
+	}
+
+	if config.parentOverride != "" {
+		fmt.Printf("Overriding parent node to '%s'\n", config.parentOverride)
+		parentNode = config.parentOverride
+	}
 
 	// Get a reference to the top level node we specified earlier
 	vislice, ok := o.Get(parentNode)
 	if !ok {
 		fmt.Printf("Missing key for multitype array")
 	}
+
 	// Get an interface that we can work with to access the sub elements
 	vslice := vislice.([]interface{})
 
+	// Check that the first element is what we expected
+	if _, ok := vslice[0].(orderedmap.OrderedMap); !ok {
+		error := fmt.Sprintf("Type of first vslice element is not an ordered map. It appears to be '%v'\n", reflect.TypeOf(vslice[0]))
+		panic(error)
+	}
 	// Get the keys for the first element so we know what the column names will be
 	columnMap := vslice[0].(orderedmap.OrderedMap)
+	//fmt.Printf("First vslice element is an ordered map\n")
 	columnNames := columnMap.Keys()
-	fmt.Printf("Creating excel workbook with following headings : '%v'\n", columnNames)
+
+	// Check if xlsx file exists already, and if it does then open and append data
+	if fileExists(outputFilename) {
+		if overwriteFile {
+			// Delete existing file
+			fmt.Printf("Overwrite flag is set, removing existing file '%s'\n", outputFilename)
+			err = os.Remove(outputFilename)
+			if err != nil {
+				panic(err)
+			}
+			// Create new file
+			xlsx = createWorkbook(worksheetName, outputFilename)
+
+		} else {
+			xlsx = modifyWorkbook(worksheetName, outputFilename)
+		}
+
+	} else {
+		xlsx = createWorkbook(worksheetName, outputFilename)
+	}
+
+	// Run code to add column names to the first row of the workbook
+	createHeadingRow(xlsx, worksheetName, columnNames)
 
 	// Set the style for the header values
 	// Just handling bold for now but we can do other styles too as per https://xuri.me/excelize/en/style.html#NewStyle
+	fmt.Printf("Bolding top row : %v\n", boldTopRow)
 	headerStyle, err2 := xlsx.NewStyle(&excelize.Style{
 		Font: &excelize.Font{
-			Bold: boldTopRow,
+			Bold: true,
 		},
 	})
 	if err2 != nil {
 		fmt.Printf("Error generating header style : '%s'\n", err2)
 	}
 
-	row = 1
-	column = 1
-
 	// Set the style
-	err = xlsx.SetRowStyle(worksheetName, row, row, headerStyle)
-	if err != nil {
-		fmt.Printf("Error setting header style : '%s'\n", err)
-	}
-
-	// Add the header row
-	for i := 0; i < len(columnNames); i++ {
-		cell, _ = excelize.CoordinatesToCellName(column, row)
-		fmt.Printf("Setting cell %s to value %s\n", cell, columnNames[i])
-		xlsx.SetCellValue(worksheetName, cell, columnNames[i])
-		//xlsx.SetCellStyle(worksheetName, cell, cell, headerStyle)
-		column++
+	if boldTopRow {
+		err = xlsx.SetRowStyle(worksheetName, 1, 1, headerStyle)
+		if err != nil {
+			fmt.Printf("Error setting header style : '%s'\n", err)
+		}
 	}
 
 	// Freeze top row if requested, see https://xuri.me/excelize/en/utils.html#SetPanes
@@ -176,7 +226,7 @@ func main() {
 
 	// Handle autofilter
 	if autoFilter {
-		// cell is still a reference to the last cell in the header row
+		cell, _ = excelize.CoordinatesToCellName(len(columnNames), 1)
 		filterRange := "A1:" + cell
 		fmt.Printf("Setting autofilter to range '%s'\n", filterRange)
 		// As per docs any filters applied need to be manually processed by us (eg hiding rows with blanks)
@@ -202,29 +252,51 @@ func main() {
 		// Each iteration should start back at column 1
 		column = 1
 
-		// Print each key-value pair contained in this slice
+		// Get the key-value pairs contained in this slice
 		vmap := v.(orderedmap.OrderedMap)
 		k := vmap.Keys()
 
-		for j := range k {
-			//a = string(asciiValue)
-			//cell = a + strconv.Itoa(2+i)
-			cell, _ = excelize.CoordinatesToCellName(column, row)
+		if len(config.keyOrder) > 0 {
+			// If we have a specified order for the values then use that
 
-			e, _ := vmap.Get(k[j])
-			//fmt.Printf("Setting cell %s to value %v\n", cell, e)
-			xlsx.SetCellValue(worksheetName, cell, e)
+			for j := 0; j < len(config.keyOrder); j++ {
+				cell, _ = excelize.CoordinatesToCellName(column, row)
 
-			// Move to the next column
-			//asciiValue++
-			column++
+				e, _ := vmap.Get(config.keyOrder[j])
+				//fmt.Printf("Setting cell %s to value %v\n", cell, e)
+				xlsx.SetCellValue(worksheetName, cell, e)
+
+				// Move to the next column
+				column++
+			}
+		} else {
+			// Otherwise use the order the json was in
+			for j := range k {
+				cell, _ = excelize.CoordinatesToCellName(column, row)
+
+				e, _ := vmap.Get(k[j])
+				//fmt.Printf("Setting cell %s to value %v\n", cell, e)
+				xlsx.SetCellValue(worksheetName, cell, e)
+
+				// Move to the next column
+				column++
+			}
 		}
+
 		//fmt.Printf("k: %v\n", k)
 
 		// Move to next row
 		row++
 	}
 
+	// Perform any post processing now that the data exists
+	if autoWidth {
+		err = SetColAutoWidth(xlsx, worksheetName)
+		if err != nil {
+			fmt.Printf("Error setting auto width : '%s'\n", err)
+		}
+	}
+
 	// Close off the file
 	if err := xlsx.SaveAs(outputFilename); err != nil {
 		log.Fatal(err)
@@ -260,6 +332,135 @@ func fileExists(filename string) bool {
 	return !info.IsDir()
 }
 
+// From https://dev.to/takakd/go-safe-truncate-string-9h0#comment-1hp14
+func TruncateString(str string, length int) string {
+	if length <= 0 {
+		return ""
+	}
+
+	if utf8.RuneCountInString(str) < length {
+		return str
+	}
+
+	return string([]rune(str)[:length])
+}
+
+func createWorkbook(worksheetName string, outputFilename string) *excelize.File {
+	fmt.Printf("Creating output spreadsheet '%s'\n", outputFilename)
+	var err error
+	// Create the file
+	xlsx := excelize.NewFile()
+
+	// Rename the default Sheet1 to this worksheet name
+	if worksheetName != "Sheet1" {
+		fmt.Printf("Renaming default worksheet to '%s'\n", worksheetName)
+		err = xlsx.SetSheetName("Sheet1", worksheetName)
+		if err != nil {
+			fmt.Printf("Error setting sheet name to '%s': %s\n", worksheetName, err)
+		}
+	}
+
+	// Set the document properties
+	err = xlsx.SetDocProps(&excelize.DocProperties{
+		Creator: "json2excel",
+		Created: time.Now().Format(time.RFC3339),
+	})
+	if err != nil {
+		fmt.Printf("Error setting document properties: %s\n", err)
+	}
+	return xlsx
+}
+
+func modifyWorkbook(worksheetName string, outputFilename string) *excelize.File {
+	fmt.Printf("Output spreadsheet '%s' already exists.\n", outputFilename)
+	xlsx, err := excelize.OpenFile(outputFilename)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	// Since we have an existing workbook, check if the sheet we want to write to already exists
+	sheetFound := false
+	for index, name := range xlsx.GetSheetMap() {
+		if name == worksheetName {
+			fmt.Printf("Found worksheet '%s' at index '%d'\n", worksheetName, index)
+			sheetFound = true
+		}
+	}
+	if !sheetFound {
+		// Create the sheet
+		fmt.Printf("Creating worksheet '%s'\n", worksheetName)
+		sheetIndex, err := xlsx.NewSheet(worksheetName)
+		if err != nil {
+			fmt.Printf("Error creating worksheet '%s' : %s\n", worksheetName, err)
+		}
+		// Set active worksheet
+		fmt.Printf("Setting active sheet to index %d", sheetIndex)
+		xlsx.SetActiveSheet(sheetIndex)
+	}
+	return xlsx
+}
+
+func createHeadingRow(xlsx *excelize.File, worksheetName string, columnNames []string) {
+	var cell string
+	row := 1
+	column := 1
+
+	if len(config.keyOrder) > 0 {
+		fmt.Printf("Creating excel workbook with heading order as per config key-order\n")
+		// Check that the number of specified columns matches input data
+		if len(config.keyOrder) != len(columnNames) {
+			error := fmt.Sprintf("Column order specified in json key-order but mismatch found in json data. %d specifed columns does not match %d found columns.", len(config.keyOrder), len(columnNames))
+			panic(error)
+		}
+
+		// Iterate the map and add the columns as per that order
+		for i := 0; i < len(config.keyOrder); i++ {
+			cell, _ = excelize.CoordinatesToCellName(column, row)
+			fmt.Printf("Setting cell %s to value %s at index %d\n", cell, config.keyOrder[i], i)
+			xlsx.SetCellValue(worksheetName, cell, config.keyOrder[i])
+			column++
+		}
+	} else {
+		fmt.Printf("Creating excel workbook with following headings : '%v'\n", columnNames)
+		// Add the header row
+		for i := 0; i < len(columnNames); i++ {
+			cell, _ = excelize.CoordinatesToCellName(column, row)
+			fmt.Printf("Setting cell %s to value %s\n", cell, columnNames[i])
+			xlsx.SetCellValue(worksheetName, cell, columnNames[i])
+			//xlsx.SetCellStyle(worksheetName, cell, cell, headerStyle)
+			column++
+		}
+	}
+
+}
+
+// Taken from https://github.com/qax-os/excelize/issues/92#issuecomment-821578446
+func SetColAutoWidth(xlsx *excelize.File, sheetName string) error {
+	// Autofit all columns according to their text content
+	cols, err := xlsx.GetCols(sheetName)
+	if err != nil {
+		return err
+	}
+	for idx, col := range cols {
+		largestWidth := 0
+		for _, rowCell := range col {
+			cellWidth := utf8.RuneCountInString(rowCell) + 2 // + 2 for margin
+			if cellWidth > largestWidth {
+				largestWidth = cellWidth
+			}
+		}
+		fmt.Printf("SetColAutoWidth calculated largest width for column index '%d' is '%d'\n", idx, largestWidth)
+		name, err := excelize.ColumnNumberToName(idx + 1)
+		if err != nil {
+			return err
+		}
+		xlsx.SetColWidth(sheetName, name, name, float64(largestWidth))
+	}
+	// No errors at this point
+	return nil
+}
+
 func TestUnmarshalJSON() {
 	s := `{
   "number": 4,