move parsers to internal library

many fixes related to previous changes
added import command

cmd/gotelem/cli/client.go

@@ -1,21 +1,148 @@
 package cli
 
 import (
+	"bufio"
+	"errors"
 	"fmt"
+	"io"
+	"os"
+	"strings"
 
-	"github.com/kschamplin/gotelem"
+	"github.com/kschamplin/gotelem/internal/db"
+	"github.com/kschamplin/gotelem/internal/logparsers"
+	"github.com/kschamplin/gotelem/skylab"
 	"github.com/urfave/cli/v2"
 )
 
+var parsersString string
+
 func init() {
 	subCmds = append(subCmds, clientCmd)
+	parsersString = func() string {
+		// create a string like "'telem', 'candump', 'anotherparser'"
+		keys := make([]string, len(logparsers.ParsersMap))
+		i := 0
+		for k := range logparsers.ParsersMap {
+			keys[i] = k
+			i++
+		}
+		s := strings.Join(keys, "', '")
+		return "'" + s + "'"
+	}()
+}
+
+var importCmd = &cli.Command{
+	Name:      "import",
+	Aliases:   []string{"i"},
+	Usage:     "import a log file into a database",
+	ArgsUsage: "[log file]",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "format",
+			Aliases: []string{"f"},
+			Usage:   "the format of the log file. One of " + parsersString,
+			Value:   "telem",
+		},
+		&cli.PathFlag{
+			Name:    "database",
+			Aliases: []string{"d", "db"},
+			Usage:   "the path of the database",
+			Value:   "./gotelem.db",
+		},
+		&cli.UintFlag{
+			Name:  "batch-size",
+			Usage: "the maximum size of each SQL transaction",
+			Value: 50,
+		},
+	},
+	Action: importAction,
+}
+
+func importAction(ctx *cli.Context) error {
+	path := ctx.Args().Get(0)
+	if path == "" {
+		fmt.Println("missing log file!")
+		cli.ShowAppHelpAndExit(ctx, -1)
+	}
+	fstream, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	fReader := bufio.NewReader(fstream)
+
+	pfun, ok := logparsers.ParsersMap[ctx.String("format")]
+	if !ok {
+		fmt.Println("invalid format provided: must be one of " + parsersString)
+		cli.ShowAppHelpAndExit(ctx, -1)
+	}
+
+	dbPath := ctx.Path("database")
+	db, err := db.OpenTelemDb(dbPath)
+	if err != nil {
+		return fmt.Errorf("error opening database: %w", err)
+	}
+
+	// we should batch data, avoiding individual transactions to the database.
+	bSize := ctx.Uint("batch-size")
+	eventsBatch := make([]*skylab.BusEvent, bSize)
+
+	batchIdx := 0
+
+	// stats for imports
+	n_packets := 0
+	n_unknown := 0
+	n_error := 0
+	for {
+		line, err := fReader.ReadString('\n')
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				break // end of file, go to the flush sequence
+			}
+			return err
+		}
+		f, err := pfun(line)
+		var idErr *skylab.UnknownIdError
+		if errors.As(err, &idErr) {
+			// unknown id
+			fmt.Printf("unknown id %v\n", idErr.Error())
+			n_unknown++
+			continue
+		} else if err != nil {
+			// TODO: we should consider absorbing all errors.
+			fmt.Printf("got an error %v\n", err)
+			n_error++
+			continue
+		}
+		n_packets++
+		eventsBatch[batchIdx] = f
+		batchIdx++
+		if batchIdx >= int(bSize) {
+			// flush it!!!!
+			err = db.AddEventsCtx(ctx.Context, eventsBatch...)
+			if err != nil {
+				fmt.Printf("error adding to database %v\n", err)
+			}
+			batchIdx = 0 // reset the batch
+		}
+
+	}
+	// check if we have remaining packets and flush them
+	if batchIdx > 0 {
+		err = db.AddEventsCtx(ctx.Context, eventsBatch[:batchIdx]...) // note the slice here!
+		if err != nil {
+			fmt.Printf("error adding to database %v\n", err)
+		}
+	}
+	fmt.Printf("import status: %d successful, %d unknown, %d errors\n", n_packets, n_unknown, n_error)
+
+	return nil
 }
 
 var clientCmd = &cli.Command{
-	Name:      "client",
+	Name:        "client",
-	Aliases:   []string{"c"},
+	Aliases:     []string{"c"},
-	Usage:     "interact with a gotelem server",
+	Subcommands: []*cli.Command{importCmd},
-	ArgsUsage: "[server url]",
+	Usage:       "Client utilities and tools",
 	Flags: []cli.Flag{
 		&cli.BoolFlag{
 			Name:    "gui",
@@ -24,13 +151,11 @@ var clientCmd = &cli.Command{
 		},
 	},
 	Description: `
-Connects to a gotelem server or relay. Can be used to 
+Connects to a gotelem server or relay. Also acts as a helper command line tool.
 	`,
 	Action: client,
 }
 
-
 func client(ctx *cli.Context) error {
 	return nil
 }
-

cmd/gotelem/cli/server.go

@@ -32,9 +32,10 @@ var serveFlags = []cli.Flag{
 		Usage:       "file to store log to",
 	},
 	&cli.PathFlag{
-		Name:  "db",
+		Name:        "db",
-		Value: "gotelem.db",
+		Aliases:     []string{"d"},
-		Usage: "database to serve",
+		DefaultText: "gotelem.db",
+		Usage:       "database to serve, if not specified will use memory",
 	},
 }
 
@@ -161,7 +162,6 @@ func (r *rpcService) Start(ctx *cli.Context, deps svcDeps) error {
 }
 
 func handleCon(conn net.Conn, broker *gotelem.Broker, l *slog.Logger, done <-chan struct{}) {
-	//	reader := msgp.NewReader(conn)
 
 	subname := fmt.Sprint("tcp", conn.RemoteAddr().String())
 
@@ -353,7 +353,7 @@ func (d *dbLoggingService) Start(cCtx *cli.Context, deps svcDeps) (err error) {
 	for {
 		select {
 		case msg := <-rxCh:
-			tdb.AddEventsCtx(cCtx.Context, msg)
+			tdb.AddEventsCtx(cCtx.Context, &msg)
 		case <-cCtx.Done():
 			return
 		}

cmd/gotelem/cli/socketcan.go

@@ -5,7 +5,7 @@ package cli
 import (
 	"time"
 
-	"github.com/kschamplin/gotelem"
+	"github.com/kschamplin/gotelem/internal/can"
 	"github.com/kschamplin/gotelem/skylab"
 	"github.com/kschamplin/gotelem/socketcan"
 	"github.com/urfave/cli/v2"
@@ -77,7 +77,7 @@ func (s *socketCANService) Start(cCtx *cli.Context, deps svcDeps) (err error) {
 	defer broker.Unsubscribe("socketCAN")
 
 	// make a channel to receive socketCAN frames.
-	rxCan := make(chan gotelem.Frame)
+	rxCan := make(chan can.Frame)
 
 	go func() {
 		for {
@@ -89,27 +89,25 @@ func (s *socketCANService) Start(cCtx *cli.Context, deps svcDeps) (err error) {
 		}
 	}()
 
-	var frame gotelem.Frame
+	var frame can.Frame
 	for {
 		select {
 		case msg := <-rxCh:
 
-			id, d, _ := skylab.ToCanFrame(msg.Data)
+			frame, err = skylab.ToCanFrame(msg.Data)
 
-			frame.Id = id
-			frame.Data = d
 
 			s.sock.Send(&frame)
 
 		case msg := <-rxCan:
-			p, err := skylab.FromCanFrame(msg.Id, msg.Data)
+			p, err := skylab.FromCanFrame(msg)
 			if err != nil {
 				logger.Warn("error parsing can packet", "id", msg.Id)
 				continue
 			}
 			cde := skylab.BusEvent{
 				Timestamp: time.Now(),
-				Id:        msg.Id,
+				Name: p.String(),
 				Data:      p,
 			}
 			broker.Publish("socketCAN", cde)

cmd/skylabify/skylabify.go

@@ -2,19 +2,15 @@ package main
 
 import (
 	"bufio"
-	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"os"
-	"regexp"
-	"strconv"
 	"strings"
 	"syscall"
-	"time"
 
-	"github.com/kschamplin/gotelem/internal/can"
+	"github.com/kschamplin/gotelem/internal/logparsers"
 	"github.com/kschamplin/gotelem/skylab"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/exp/slog"
@@ -46,6 +42,17 @@ required for piping candump into skylabify. Likewise, data should be stored with
 -l.
 
 `
+	parsersString := func() string {
+		// create a string like "'telem', 'candump', 'anotherparser'"
+		keys := make([]string, len(logparsers.ParsersMap))
+		i := 0
+		for k := range logparsers.ParsersMap {
+			keys[i] = k
+			i++
+		}
+		s := strings.Join(keys, "', '")
+		return "'" + s + "'"
+	}()
 
 	app.Flags = []cli.Flag{
 		&cli.BoolFlag{
@@ -55,7 +62,7 @@ required for piping candump into skylabify. Likewise, data should be stored with
 		&cli.StringFlag{
 			Name:    "format",
 			Aliases: []string{"f"},
-			Usage:   "the format of the incoming data. One of 'telem', 'candump'",
+			Usage:   "the format of the incoming data. One of " + parsersString,
 		},
 	}
 
@@ -65,150 +72,6 @@ required for piping candump into skylabify. Likewise, data should be stored with
 	}
 }
 
-// A FormatError is an error when parsing a format. Typically we simply ignore
-// these and move on, but they can optionally wrap another error that is fatal.
-type FormatError struct {
-	msg string
-	err error
-}
-
-func (e *FormatError) Error() string {
-	return fmt.Sprintf("%s:%s", e.msg, e.err.Error())
-}
-func (e *FormatError) Unwrap() error {
-	return e.err
-}
-
-func NewFormatError(msg string, err error) error {
-	return &FormatError{msg: msg, err: err}
-}
-
-// A Parser takes a string containing one line of a particular log file
-// and returns an associated skylab.BusEvent representing the packet.
-// if no packet is found, an error is returned instead.
-type ParserFunc func(string) (skylab.BusEvent, error)
-
-func parseCanDumpLine(dumpLine string) (b skylab.BusEvent, err error) {
-	// dumpline looks like this:
-	// (1684538768.521889) can0 200#8D643546
-	// remove trailing newline
-	dumpLine = strings.TrimSpace(dumpLine)
-	segments := strings.Split(dumpLine, " ")
-
-	var unixSeconds, unixMicros int64
-	fmt.Sscanf(segments[0], "(%d.%d)", &unixSeconds, &unixMicros)
-	b.Timestamp = time.Unix(unixSeconds, unixMicros)
-
-	// now we extract the remaining data:
-	hexes := strings.Split(segments[2], "#") // first portion is id, second is data
-
-	id, err := strconv.ParseUint(hexes[0], 16, 64)
-	if err != nil {
-		err = NewFormatError("failed to parse id", err)
-		return
-	}
-	if (len(hexes[1]) % 2) != 0 {
-		err = NewFormatError("odd number of hex characters", nil)
-		return
-	}
-	rawData, err := hex.DecodeString(hexes[1])
-	if err != nil {
-		err = NewFormatError("failed to decode hex data", err)
-		return
-	}
-
-	frame := can.Frame{
-		// TODO: fix extended ids. we assume not extended for now.
-		Id:   can.CanID{Id: uint32(id), Extended: false},
-		Data: rawData,
-		Kind: can.CanDataFrame,
-	}
-
-	b.Data, err = skylab.FromCanFrame(frame)
-
-	if err != nil {
-		err = NewFormatError("failed to parse can frame", err)
-		return
-	}
-
-	// set the name
-	b.Name = b.Data.String()
-
-	return
-}
-
-func parseTelemLogLine(line string) (b skylab.BusEvent, err error) {
-	// strip trailng newline since we rely on it being gone
-	line = strings.TrimSpace(line)
-	// data is of the form
-	// 1698180835.318 0619D80564080EBE241
-	// the second part there is 3 nibbles (12 bits, 3 hex chars) for can ID,
-	// the rest is data.
-	// this regex does the processing.
-	r := regexp.MustCompile(`^(\d+).(\d{3}) (\w{3})(\w+)$`)
-
-	// these files tend to get corrupted. there are all kinds of nasties that can happen.
-	// defense against random panics
-	defer func() {
-		if r := recover(); r != nil {
-			err = NewFormatError("caught panic", nil)
-		}
-	}()
-	a := r.FindStringSubmatch(line)
-	if a == nil {
-		err = NewFormatError("no regex match", nil)
-		return
-	}
-	var unixSeconds, unixMillis int64
-	// note that a contains 5 elements, the first being the full match.
-	// so we start from the second element
-	unixSeconds, err = strconv.ParseInt(a[1], 10, 0)
-	if err != nil {
-		err = NewFormatError("failed to parse unix seconds", err)
-		return
-	}
-	unixMillis, err = strconv.ParseInt(a[2], 10, 0)
-	if err != nil {
-		err = NewFormatError("failed to parse unix millis", err)
-		return
-	}
-	ts := time.Unix(unixSeconds, unixMillis*1e6)
-
-	id, err := strconv.ParseUint(a[3], 16, 16)
-	if err != nil {
-		err = NewFormatError("failed to parse id", err)
-		return
-	}
-
-	if len(a[4])%2 != 0 {
-		// odd hex chars, protect against a panic
-		err = NewFormatError("wrong amount of hex chars", nil)
-	}
-	rawData, err := hex.DecodeString(a[4])
-	if err != nil {
-		err = NewFormatError("failed to parse hex data", err)
-		return
-	}
-	frame := can.Frame{
-		Id:   can.CanID{Id: uint32(id), Extended: false},
-		Data: rawData,
-		Kind: can.CanDataFrame,
-	}
-	b.Timestamp = ts
-	b.Data, err = skylab.FromCanFrame(frame)
-	if err != nil {
-		err = NewFormatError("failed to parse can frame", err)
-		return
-	}
-	b.Name = b.Data.String()
-
-	return
-}
-
-var parseMap = map[string]ParserFunc{
-	"telem":   parseTelemLogLine,
-	"candump": parseCanDumpLine,
-}
 
 func run(ctx *cli.Context) (err error) {
 	path := ctx.Args().Get(0)
@@ -229,9 +92,9 @@ func run(ctx *cli.Context) (err error) {
 
 	fileReader := bufio.NewReader(istream)
 
-	var pfun ParserFunc
+	var pfun logparsers.ParserFunc
 
-	pfun, ok := parseMap[ctx.String("format")]
+	pfun, ok := logparsers.ParsersMap[ctx.String("format")]
 	if !ok {
 		fmt.Println("invalid format!")
 		cli.ShowAppHelpAndExit(ctx, int(syscall.EINVAL))

frame_kind.go

@@ -1,26 +0,0 @@
-// Code generated by "stringer -output=frame_kind.go -type Kind"; DO NOT EDIT.
-
-package gotelem
-
-import "strconv"
-
-func _() {
-	// An "invalid array index" compiler error signifies that the constant values have changed.
-	// Re-run the stringer command to generate them again.
-	var x [1]struct{}
-	_ = x[CanSFFFrame-0]
-	_ = x[CanEFFFrame-1]
-	_ = x[CanRTRFrame-2]
-	_ = x[CanErrFrame-3]
-}
-
-const _Kind_name = "CanSFFFrameCanEFFFrameCanRTRFrameCanErrFrame"
-
-var _Kind_index = [...]uint8{0, 11, 22, 33, 44}
-
-func (i Kind) String() string {
-	if i >= Kind(len(_Kind_index)-1) {
-		return "Kind(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-	return _Kind_name[_Kind_index[i]:_Kind_index[i+1]]
-}

http.go

@@ -46,8 +46,6 @@ func TelemRouter(log *slog.Logger, broker *Broker, db *db.TelemDb) http.Handler
 	// To future residents - you can add new API calls/systems in /api/v2
 	// Don't break anything in api v1! keep legacy code working!
 
-	// serve up a local status page.
-
 	return r
 }
 
@@ -69,7 +67,7 @@ func apiV1(broker *Broker, db *db.TelemDb) chi.Router {
 	r.Route("/packets", func(r chi.Router) {
 		r.Get("/subscribe", apiV1PacketSubscribe(broker, db))
 		r.Post("/", func(w http.ResponseWriter, r *http.Request) {
-			var pkgs []skylab.BusEvent
+			var pkgs []*skylab.BusEvent
 			decoder := json.NewDecoder(r.Body)
 			if err := decoder.Decode(&pkgs); err != nil {
 				w.WriteHeader(http.StatusTeapot)
@@ -111,7 +109,7 @@ func apiV1(broker *Broker, db *db.TelemDb) chi.Router {
 
 // apiV1Subscriber is a websocket session for the v1 api.
 type apiV1Subscriber struct {
-	idFilter []uint32 // list of Ids to subscribe to. If it's empty, subscribes to all.
+	nameFilter []string // names of packets we care about.
 }
 
 // this is a websocket stream.
@@ -143,13 +141,14 @@ func apiV1PacketSubscribe(broker *Broker, db *db.TelemDb) http.HandlerFunc {
 			case <-r.Context().Done():
 				return
 			case msgIn := <-sub:
-				if len(sess.idFilter) == 0 {
+				if len(sess.nameFilter) == 0 {
 					// send it.
 					wsjson.Write(r.Context(), c, msgIn)
 				}
-				for _, id := range sess.idFilter {
+				for _, name := range sess.nameFilter {
-					if id == msgIn.Id {
+					if name == msgIn.Name {
 						// send it
+						wsjson.Write(r.Context(), c, msgIn)
 						break
 					}
 				}

internal/db/db.go

@@ -7,7 +7,6 @@ import (
 	"database/sql"
 	"encoding/json"
 	"fmt"
-	"strconv"
 	"strings"
 	"time"
 
@@ -72,11 +71,11 @@ func (tdb *TelemDb) SetVersion(version int) error {
 
 // sql expression to insert a bus event into the packets database.1
 const sqlInsertEvent = `
-INSERT INTO "bus_events" (ts, name, data) VALUES ($1, $2, json($3));
+INSERT INTO "bus_events" (ts, name, data) VALUES
 `
 
 // AddEvent adds the bus event to the database.
-func (tdb *TelemDb) AddEventsCtx(ctx context.Context, events ...skylab.BusEvent) (err error) {
+func (tdb *TelemDb) AddEventsCtx(ctx context.Context, events ...*skylab.BusEvent) (err error) {
 	//
 	tx, err := tdb.db.BeginTx(ctx, nil)
 	if err != nil {
@@ -118,12 +117,11 @@ func (tdb *TelemDb) AddEventsCtx(ctx context.Context, events ...skylab.BusEvent)
 	return
 }
 
-func (tdb *TelemDb) AddEvents(events ...skylab.BusEvent) (err error) {
+func (tdb *TelemDb) AddEvents(events ...*skylab.BusEvent) (err error) {
 
 	return tdb.AddEventsCtx(context.Background(), events...)
 }
 
-
 /// Query fragment guide:
 /// We need to be able to easily construct safe(!) and meaningful queries programatically
 /// so we make some new types that can be turned into SQL fragments that go inside the where clause.
@@ -198,7 +196,7 @@ func (tdb *TelemDb) GetEvents(limit int, where ...QueryFrag) (events []skylab.Bu
 
 		BusEv := skylab.BusEvent{
 			Timestamp: time.UnixMilli(int64(ev.Timestamp)),
-			Name:       ev.Name,
+			Name:      ev.Name,
 		}
 		BusEv.Data, err = skylab.FromJson(ev.Name, ev.Data)
 

internal/logparsers/parsers.go

@@ -0,0 +1,168 @@
+package logparsers
+
+import (
+	"encoding/hex"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/kschamplin/gotelem/internal/can"
+	"github.com/kschamplin/gotelem/skylab"
+)
+
+// A FormatError is an error when parsing a format. Typically we simply ignore
+// these and move on, but they can optionally wrap another error that is fatal.
+type FormatError struct {
+	msg string
+	err error
+}
+
+func (e *FormatError) Error() string {
+	if e.err != nil {
+		return fmt.Sprintf("%s:%s", e.msg, e.err.Error())
+	}
+	return fmt.Sprintf("%s", e.msg)
+
+}
+func (e *FormatError) Unwrap() error {
+	return e.err
+}
+
+func NewFormatError(msg string, err error) error {
+	return &FormatError{msg: msg, err: err}
+}
+
+// A Parser takes a string containing one line of a particular log file
+// and returns an associated skylab.BusEvent representing the packet.
+// if no packet is found, an error is returned instead.
+type ParserFunc func(string) (*skylab.BusEvent, error)
+
+func parseCanDumpLine(dumpLine string) (b *skylab.BusEvent, err error) {
+	// dumpline looks like this:
+	// (1684538768.521889) can0 200#8D643546
+	// remove trailing newline
+	dumpLine = strings.TrimSpace(dumpLine)
+	segments := strings.Split(dumpLine, " ")
+
+	var unixSeconds, unixMicros int64
+	fmt.Sscanf(segments[0], "(%d.%d)", &unixSeconds, &unixMicros)
+
+	// now we extract the remaining data:
+	hexes := strings.Split(segments[2], "#") // first portion is id, second is data
+
+	id, err := strconv.ParseUint(hexes[0], 16, 64)
+	if err != nil {
+		err = NewFormatError("failed to parse id", err)
+		return
+	}
+	if (len(hexes[1]) % 2) != 0 {
+		err = NewFormatError("odd number of hex characters", nil)
+		return
+	}
+	rawData, err := hex.DecodeString(hexes[1])
+	if err != nil {
+		err = NewFormatError("failed to decode hex data", err)
+		return
+	}
+
+	frame := can.Frame{
+		// TODO: fix extended ids. we assume not extended for now.
+		Id:   can.CanID{Id: uint32(id), Extended: false},
+		Data: rawData,
+		Kind: can.CanDataFrame,
+	}
+
+	b = &skylab.BusEvent{
+		Timestamp: time.Unix(unixSeconds, unixMicros),
+	}
+
+	b.Data, err = skylab.FromCanFrame(frame)
+
+	if err != nil {
+		err = NewFormatError("failed to parse can frame", err)
+		return
+	}
+
+	// set the name
+	b.Name = b.Data.String()
+
+	return
+}
+
+func parseTelemLogLine(line string) (b *skylab.BusEvent, err error) {
+	// strip trailng newline since we rely on it being gone
+	line = strings.TrimSpace(line)
+	// data is of the form
+	// 1698180835.318 0619D80564080EBE241
+	// the second part there is 3 nibbles (12 bits, 3 hex chars) for can ID,
+	// the rest is data.
+	// this regex does the processing.
+	r := regexp.MustCompile(`^(\d+).(\d{3}) (\w{3})(\w+)$`)
+
+	// these files tend to get corrupted. there are all kinds of nasties that can happen.
+	// defense against random panics
+	defer func() {
+		if r := recover(); r != nil {
+			err = NewFormatError("caught panic", nil)
+		}
+	}()
+	a := r.FindStringSubmatch(line)
+	if a == nil || len(a) != 5 {
+		err = NewFormatError("no regex match", nil)
+		return
+	}
+	var unixSeconds, unixMillis int64
+	// note that a contains 5 elements, the first being the full match.
+	// so we start from the second element
+	unixSeconds, err = strconv.ParseInt(a[1], 10, 0)
+	if err != nil {
+		err = NewFormatError("failed to parse unix seconds", err)
+		return
+	}
+	unixMillis, err = strconv.ParseInt(a[2], 10, 0)
+	if err != nil {
+		err = NewFormatError("failed to parse unix millis", err)
+		return
+	}
+	ts := time.Unix(unixSeconds, unixMillis*1e6)
+
+	id, err := strconv.ParseUint(a[3], 16, 16)
+	if err != nil {
+		err = NewFormatError("failed to parse id", err)
+		return
+	}
+
+	if len(a[4])%2 != 0 {
+		// odd hex chars, protect against a panic
+		err = NewFormatError("wrong amount of hex chars", nil)
+	}
+	rawData, err := hex.DecodeString(a[4])
+	if err != nil {
+		err = NewFormatError("failed to parse hex data", err)
+		return
+	}
+	frame := can.Frame{
+		Id:   can.CanID{Id: uint32(id), Extended: false},
+		Data: rawData,
+		Kind: can.CanDataFrame,
+	}
+
+	b = &skylab.BusEvent{
+		Timestamp: ts,
+	}
+	b.Data, err = skylab.FromCanFrame(frame)
+	if err != nil {
+		err = NewFormatError("failed to parse can frame", err)
+		return
+	}
+	b.Name = b.Data.String()
+
+	return
+}
+
+var ParsersMap = map[string]ParserFunc{
+	"telem":   parseTelemLogLine,
+	"candump": parseCanDumpLine,
+}

py/pytelem/skylab_attr.py

@@ -3,6 +3,9 @@ from typing import Dict, Optional, List, Union
 from enum import Enum
 from attrs import define, field, validators
 
+# we define a validator for our names - alphanumeric and underscores
+# most things can't support numbers as the first character, so we don't either.
+name_validator = validators.matches_re(r"^[A-Za-z_][A-Za-z0-9_]?$")
 
 @define
 class Bus():
@@ -58,7 +61,7 @@ class FieldType(str, Enum):
 
 @define
 class CustomTypeDef():
-    name: str
+    name: str = field(validator=[name_validator])
     base_type: FieldType # should be a strict size
     values: Union[List[str], Dict[str, int]]
 
@@ -66,11 +69,11 @@ class CustomTypeDef():
 @define
 class BitfieldBit():
     "micro class to represent one bit in bitfields"
-    name: str
+    name: str = field(validator=[name_validator])
 
 @define
 class Field():
-    name: str = field(validator=[validators.matches_re(r"^[A-Za-z0-9_]+$")])
+    name: str = field(validator=[name_validator])
     type: FieldType
 
     #metadata
@@ -80,7 +83,7 @@ class Field():
 
 @define
 class BitField():
-    name: str = field(validator=[validators.matches_re(r"^[A-Za-z0-9_]+$")])
+    name: str = field(validator=[name_validator])
     type: str = field(default="bitfield", init=False) # it's a constant value
     bits: List[BitfieldBit]
 

skylab/skylab.go

@@ -12,6 +12,7 @@ import (
 	// this is needed so that we can run make_skylab.go
 	// without this, the yaml library will be removed
 	// when we run `go mod tidy`
+	"github.com/kschamplin/gotelem/internal/can"
 	_ "gopkg.in/yaml.v3"
 )
 
@@ -62,7 +63,7 @@ type Unmarshaler interface {
 
 // Ider is a packet that can get its ID, based on the index of the packet, if any.
 type Ider interface {
-	CANId() (uint32, error)
+	CanId() (can.CanID, error)
 }
 
 // Sizer allows for fast allocation.
@@ -71,13 +72,15 @@ type Sizer interface {
 }
 
 // CanSend takes a packet and makes CAN framing data.
-func ToCanFrame(p Packet) (id uint32, data []byte, err error) {
+func ToCanFrame(p Packet) (f can.Frame, err error) {
 
-	id, err = p.CANId()
+
+	f.Id, err = p.CanId()
 	if err != nil {
 		return
 	}
-	data, err = p.MarshalPacket()
+	f.Data, err = p.MarshalPacket()
+	f.Kind = can.CanDataFrame
 	return
 }
 

skylab/templates/golang.go.tmpl

@@ -45,15 +45,17 @@ type {{$structName}} struct {
 {{- end }}
 }
 
-func (p *{{$structName}}) CANId() (uint32, error) {
+func (p *{{$structName}}) CanId() (can.CanID, error) {
+	c := can.CanID{Extended: {{.Extended}}}
 {{- if .Repeat }}
 	if p.Idx >= {{.Repeat}} {
-		return 0, &UnknownIdError{ {{ printf "0x%X" .Id }} }
+		return c, &UnknownIdError{ {{ printf "0x%X" .Id }} }
 	}
-	return {{ printf "0x%X" .Id }} + p.Idx, nil
+	c.Id = {{ printf "0x%X" .Id }} + p.Idx
 {{- else }}
-	return {{ printf "0x%X" .Id }}, nil
+	c.Id = {{ printf "0x%X" .Id }}
 {{- end }}
+	return c, nil
 }
 
 func (p *{{$structName}}) Size() uint {
@@ -107,10 +109,10 @@ var idMap = map[can.CanID]bool{
 	{{ range $p := .Packets -}}
 	{{ if $p.Repeat }} 
 	{{ range $idx := Nx (int $p.Id) $p.Repeat $p.Offset -}}
-	can.CanID{ Id: {{ $idx | printf "0x%X"}}, Extended: {{$p.Extended}} }: true,
+	{ Id: {{ $idx | printf "0x%X"}}, Extended: {{$p.Extended}} }: true,
 	{{ end }}
 	{{- else }}
-	can.CanID{ Id: {{ $p.Id | printf "0x%X" }}, Extended: {{$p.Extended}} }: true,
+	{ Id: {{ $p.Id | printf "0x%X" }}, Extended: {{$p.Extended}} }: true,
 	{{- end}}
 	{{- end}}
 }