From 1570497ddc7f928d71e2bd5dc532bd75d1ee3155 Mon Sep 17 00:00:00 2001 From: saji Date: Fri, 7 Jul 2023 14:03:53 -0500 Subject: [PATCH] add old log fixer. --- cmd/fixer/fixer.go | 205 +++++++++++++++++++++++++++++++++++++ cmd/gotelem/cli/server.go | 1 - cmd/skylabify/skylabify.go | 3 +- readme.md | 5 +- 4 files changed, 211 insertions(+), 3 deletions(-) create mode 100644 cmd/fixer/fixer.go diff --git a/cmd/fixer/fixer.go b/cmd/fixer/fixer.go new file mode 100644 index 0000000..4dbbf33 --- /dev/null +++ b/cmd/fixer/fixer.go @@ -0,0 +1,205 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "syscall" + "time" + + "github.com/kschamplin/gotelem/skylab" + "github.com/urfave/cli/v2" + "golang.org/x/exp/slog" +) + +// fixer resolves four major issues with CAN dumps: +// 1. ISO8601 timestamps +// 2. Unix seconds timestamps +// 3. Missing/broken timestamps (retime) +// 4. missing names + +func main() { + app := cli.NewApp() + app.Name = "fixer" + app.Usage = " fix skylabify outputs" + app.ArgsUsage = "" + app.Description = `fixer fixes four major issues with CAN dumps +1. ISO8601 timestamps --time iso8601 +2. Unix seconds timestamps --time seconds +3. Missing/broken timestamps --retime +4. missing names (enabled by default, --no-rename to skip) + ` + app.Flags = []cli.Flag{ + &cli.BoolFlag{ + Name: "retime", + Usage: "ignore timestamps and retime data based on 200ms heartbeat packet", + }, + &cli.StringFlag{ + Name: "format", + Usage: "Timestamp format. One of 'iso8601', 'seconds'. Ignored if using retime.", + }, + &cli.BoolFlag{ + Name: "no-rename", + Usage: "skip changing packet names", + }, + &cli.StringFlag{ + Name: "output", + Usage: "file to output to. defaults to stdout.", + }, + } + + app.Before = validateArgs + app.Action = run + app.HideHelp = true + app.Run(os.Args) +} + +var allowedFormats = []string{ + "iso8601", + "seconds", +} + +func checkFormat(input string) bool { + for _, allowed := range allowedFormats { + if input == allowed { + return true + } + } + return false +} + +func validateArgs(ctx *cli.Context) error { + if ctx.IsSet("format") { + format := ctx.String("format") + if !checkFormat(format) { + fmt.Printf("invalid format string, got %s, must be one of %s", format, strings.Join(allowedFormats, ", ")) + cli.ShowAppHelpAndExit(ctx, int(syscall.EINVAL)) + } + + } + if ctx.Args().Get(0) == "" { + fmt.Println("missing input file") + cli.ShowAppHelpAndExit(ctx, int(syscall.EINVAL)) + } + return nil +} + +func run(ctx *cli.Context) (err error) { + path := ctx.Args().Get(0) + + var istream *os.File + if path == "-" { + istream = os.Stdin + } else { + istream, err = os.Open(path) + if err != nil { + return + } + } + var ostream *os.File + oFilename := ctx.Args().Get(1) + if oFilename == "" { + ostream = os.Stdout + } else { + ostream, err = os.Create(oFilename) + if err != nil { + return + } + } + + iReader := json.NewDecoder(istream) + + shouldRetime := ctx.Bool("retime") + + // used for retime - increment by 200 ms every time we see a WsrStatusPacket + currentTick := time.Now().UnixMilli() + + for { + // read a line of json and fix names if we should. + + // FIXME: handle missing trailing newline. + // based on the fomat string, we should parse the data raw... + var res brokenCANMsg + err := iReader.Decode(&res) + + // float64 can have issues when represeneting + // unix milliseconds. the JSON decoder supports + // a "Number" struct that can be either. + iReader.UseNumber() + if err != nil { + return err + } + + var goodPkt skylab.RawJsonEvent + + goodPkt.Data = res.Data + goodPkt.Id = uint32(res.Id) + goodPkt.Name = res.Name + // wait to decode packet before rename. + + switch ts := res.Timestamp.(type) { + case json.Number: + // if it contains a decimal. + if strings.Contains(ts.String(), ".") { + // it's a float. + t, _ := ts.Float64() + t = t * 1000 + goodPkt.Timestamp = int64(t) + } else { + // it's an int. + t, _ := ts.Int64() + if ctx.String("format") == "seconds" { + t = t * 1000 + } + goodPkt.Timestamp = t + } + case string: + // parse as ISO8601 + // use unix millis + var t time.Time + err := t.UnmarshalText([]byte(ts)) + if err != nil { + panic(err) + } + goodPkt.Timestamp = t.UnixMilli() + + } + + if shouldRetime { + if goodPkt.Id == uint32(skylab.WsrStatusInformationId) { + // bump the clock 200ms + currentTick += 200 + } + goodPkt.Timestamp = currentTick + } + + // now, spit it out. + var bEv skylab.BusEvent + bEv.Timestamp = time.UnixMilli(goodPkt.Timestamp) + bEv.Id = goodPkt.Id + bEv.Data, err = skylab.FromJson(goodPkt.Id, goodPkt.Data) + var idErr *skylab.UnknownIdError + if errors.As(err, &idErr) { + // unknown id + slog.Info("unknown id", "err", err) + continue + } else if err != nil { + return err + } + + out, err := json.Marshal(&bEv) + if err != nil { + panic(err) + } + fmt.Fprintln(ostream, string(out)) + } +} + +type brokenCANMsg struct { + Timestamp any `json:"ts"` + Id int32 + Name string + Data json.RawMessage +} diff --git a/cmd/gotelem/cli/server.go b/cmd/gotelem/cli/server.go index 4cafb1d..1027acf 100644 --- a/cmd/gotelem/cli/server.go +++ b/cmd/gotelem/cli/server.go @@ -335,7 +335,6 @@ func (d *dbLoggingService) Start(cCtx *cli.Context, deps svcDeps) (err error) { for { select { case msg := <-rxCh: - deps.Logger.Info("boop", "msg", msg) tdb.AddEventsCtx(cCtx.Context, msg) case <-cCtx.Done(): return diff --git a/cmd/skylabify/skylabify.go b/cmd/skylabify/skylabify.go index 57144b5..dd91d40 100644 --- a/cmd/skylabify/skylabify.go +++ b/cmd/skylabify/skylabify.go @@ -62,9 +62,10 @@ required for piping candump into skylabify. Likewise, data should be stored with func run(ctx *cli.Context) (err error) { path := ctx.Args().Get(0) if path == "" { - fmt.Printf("missing input file\n") + fmt.Println("missing input file") cli.ShowAppHelpAndExit(ctx, int(syscall.EINVAL)) } + var istream *os.File if path == "-" { diff --git a/readme.md b/readme.md index 847f26a..ffee87d 100644 --- a/readme.md +++ b/readme.md @@ -58,4 +58,7 @@ own system, and it's a single executable to share to others with the same OS/arc ## Building -There are build tags to enable/disable certain features, like the graphical GUI. \ No newline at end of file +`gotelem` was designed to be all-inclusive while being easy to build and have good cross-platform support. Binaries are a single, +statically linked file that can be shared to other users of the same OS. Certain features, like socketCAN support, are only enabled on platforms that +support them (Linux). This is handled automatically; builds will exclude the socketCAN files and the additional commands and features will not be present in the CLI. +