From f089f3777112f72afd8aa637e01b45b86d35d7ec Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sun, 3 Nov 2019 09:09:33 +1000 Subject: [PATCH 01/40] Initial commit --- LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + 2 files changed, 203 insertions(+) create mode 100644 LICENSE create mode 100644 README.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c11852a --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# ordereddict +A simple Ordered Dict implementation. From 8f8de2613e3e7a773cc698c202fc042ee770285e Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Sun, 3 Nov 2019 10:40:36 +1000 Subject: [PATCH 02/40] Initial commit of ordered dicts. --- LICENSE | 4 +- README.txt | 14 ++ go.mod | 8 ++ go.sum | 12 ++ ordereddict.go | 340 ++++++++++++++++++++++++++++++++++++++++++++ ordereddict_test.go | 113 +++++++++++++++ 6 files changed, 489 insertions(+), 2 deletions(-) create mode 100644 README.txt create mode 100644 go.mod create mode 100644 go.sum create mode 100644 ordereddict.go create mode 100644 ordereddict_test.go diff --git a/LICENSE b/LICENSE index 261eeb9..dfadb57 100644 --- a/LICENSE +++ b/LICENSE @@ -178,7 +178,7 @@ APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" + boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright 2019 velocidex Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..a4e211a --- /dev/null +++ b/README.txt @@ -0,0 +1,14 @@ +# Ordered Dict implementation for go. + +There are a number of OrderedDict implementations out there and this +is one is less capable than a more generic implementation: + +- We do not support key deletion +- Replacing a key is O(n) with the number of keys + +Therefore this implementation is only suitable for dicts with few keys +that are not generally replaced or deleted. + +The main benefit of this implementation is that it maintains key order +when serializing/unserializing from JSON and it is very memory +efficient. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b01afb5 --- /dev/null +++ b/go.mod @@ -0,0 +1,8 @@ +module github.com/Velocidex/ordereddict + +go 1.13 + +require ( + github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 + github.com/stretchr/testify v1.4.0 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8b718bb --- /dev/null +++ b/go.sum @@ -0,0 +1,12 @@ +github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E= +github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/ordereddict.go b/ordereddict.go new file mode 100644 index 0000000..265cf15 --- /dev/null +++ b/ordereddict.go @@ -0,0 +1,340 @@ +package vfilter + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "strings" + "sync" +) + +// A concerete implementation of a row - similar to Python's +// OrderedDict. Main difference is that delete is not implemented - +// we just preserve the order of insertions. +type Dict struct { + sync.Mutex + + store map[string]interface{} + keys []string + case_map map[string]string + + default_value interface{} +} + +func NewDict() *Dict { + return &Dict{ + store: make(map[string]interface{}), + } +} + +func (self *Dict) IsCaseInsensitive() bool { + self.Lock() + defer self.Unlock() + + return self.case_map != nil +} + +func (self *Dict) MergeFrom(other *Dict) { + for _, key := range other.keys { + value, pres := other.Get(key) + if pres { + self.Set(key, value) + } + } +} + +func (self *Dict) SetDefault(value interface{}) *Dict { + self.Lock() + defer self.Unlock() + + self.default_value = value + return self +} + +func (self *Dict) GetDefault() interface{} { + self.Lock() + defer self.Unlock() + + return self.default_value +} + +func (self *Dict) SetCaseInsensitive() *Dict { + self.Lock() + defer self.Unlock() + + self.case_map = make(map[string]string) + return self +} + +func remove(s []string, r string) []string { + for i, v := range s { + if v == r { + return append(s[:i], s[i+1:]...) + } + } + return s +} + +func (self *Dict) Set(key string, value interface{}) *Dict { + self.Lock() + defer self.Unlock() + + // O(n) but for our use case this is faster since Dicts are + // typically small and we rarely overwrite a key. + _, pres := self.store[key] + if pres { + self.keys = append(remove(self.keys, key), key) + } else { + self.keys = append(self.keys, key) + } + + self.store[key] = value + + if self.case_map != nil { + self.case_map[strings.ToLower(key)] = key + } + + return self +} + +func (self *Dict) Len() int { + self.Lock() + defer self.Unlock() + + return len(self.store) +} + +func (self *Dict) Get(key string) (interface{}, bool) { + self.Lock() + defer self.Unlock() + + if self.case_map != nil { + real_key, pres := self.case_map[strings.ToLower(key)] + if pres { + key = real_key + } + } + + val, ok := self.store[key] + if !ok && self.default_value != nil { + return self.default_value, false + } + + return val, ok +} + +func (self *Dict) Keys() []string { + self.Lock() + defer self.Unlock() + + return self.keys[:] +} + +func (self *Dict) ToDict() *map[string]interface{} { + self.Lock() + defer self.Unlock() + + result := make(map[string]interface{}) + + for _, key := range self.keys { + value, pres := self.store[key] + if pres { + result[key] = value + } + } + + return &result +} + +func (self *Dict) String() string { + self.Lock() + defer self.Unlock() + + builder := make([]string, len(self.keys)) + + var index int = 0 + for _, key := range self.keys { + val, _ := self.store[key] + builder[index] = fmt.Sprintf("%v:%v, ", key, val) + index++ + } + return fmt.Sprintf("Dict%v", builder) +} + +func (self *Dict) GoString() string { + return self.String() +} + +// this implements type json.Unmarshaler interface, so can be called in json.Unmarshal(data, om) +func (self *Dict) UnmarshalJSON(data []byte) error { + self.Lock() + defer self.Unlock() + + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + + // must open with a delim token '{' + t, err := dec.Token() + if err != nil { + return err + } + delim, ok := t.(json.Delim) + if !ok || delim != '{' { + return fmt.Errorf("expect JSON object open with '{'") + } + + err = self.parseobject(dec) + if err != nil { + return err + } + + t, err = dec.Token() + if err != io.EOF { + return fmt.Errorf("expect end of JSON object but got more token: %T: %v or err: %v", t, t, err) + } + + return nil +} + +func (self *Dict) parseobject(dec *json.Decoder) (err error) { + var t json.Token + for dec.More() { + t, err = dec.Token() + if err != nil { + return err + } + + key, ok := t.(string) + if !ok { + return fmt.Errorf("expecting JSON key should be always a string: %T: %v", t, t) + } + + t, err = dec.Token() + if err == io.EOF { + break + + } else if err != nil { + return err + } + + var value interface{} + value, err = handledelim(t, dec) + if err != nil { + return err + } + self.keys = append(self.keys, key) + self.store[key] = value + if self.case_map != nil { + self.case_map[strings.ToLower(key)] = key + } + } + + t, err = dec.Token() + if err != nil { + return err + } + delim, ok := t.(json.Delim) + if !ok || delim != '}' { + return fmt.Errorf("expect JSON object close with '}'") + } + + return nil +} + +func parsearray(dec *json.Decoder) (arr []interface{}, err error) { + var t json.Token + arr = make([]interface{}, 0) + for dec.More() { + t, err = dec.Token() + if err != nil { + return + } + + var value interface{} + value, err = handledelim(t, dec) + if err != nil { + return + } + arr = append(arr, value) + } + t, err = dec.Token() + if err != nil { + return + } + delim, ok := t.(json.Delim) + + if !ok || delim != ']' { + err = fmt.Errorf("expect JSON array close with ']'") + return + } + + return +} + +func handledelim(token json.Token, dec *json.Decoder) (res interface{}, err error) { + switch t := token.(type) { + case json.Delim: + switch t { + case '{': + dict2 := NewDict() + err = dict2.parseobject(dec) + if err != nil { + return + } + return dict2, nil + case '[': + var value []interface{} + value, err = parsearray(dec) + if err != nil { + return + } + return value, nil + default: + return nil, fmt.Errorf("Unexpected delimiter: %q", t) + } + + case json.Number: + value, err := t.Int64() + if err == nil { + return value, nil + } + + float, err := t.Float64() + if err == nil { + return float, nil + } + + return nil, fmt.Errorf("Unexpected token: %v", token) + } + return token, nil +} + +func (self Dict) MarshalJSON() ([]byte, error) { + self.Lock() + defer self.Unlock() + + result := "{" + for _, k := range self.keys { + + // add key + kEscaped := strings.Replace(k, `"`, `\"`, -1) + result = result + `"` + kEscaped + `":` + + // add value + v := self.store[k] + + vBytes, err := json.Marshal(v) + + if err == nil { + result = result + string(vBytes) + "," + } else { + result = result + "null," + } + } + if len(self.keys) > 0 { + result = result[0 : len(result)-1] + } + result = result + "}" + return []byte(result), nil +} diff --git a/ordereddict_test.go b/ordereddict_test.go new file mode 100644 index 0000000..d7f8c4e --- /dev/null +++ b/ordereddict_test.go @@ -0,0 +1,113 @@ +package vfilter + +import ( + "encoding/json" + "reflect" + "testing" + "time" + + "github.com/alecthomas/repr" + "github.com/stretchr/testify/assert" +) + +type dictSerializationTest struct { + dict *Dict + serialized string +} + +var ( + dictSerializationTests = []dictSerializationTest{ + {NewDict().Set("Foo", "Bar"), `{"Foo":"Bar"}`}, + + // Test an unserilizable member - This should not prevent the + // entire dict from serializing - only that member should be + // ignored. + {NewDict().Set("Foo", "Bar"). + Set("Time", time.Unix(3000000000000000, 0)), + `{"Foo":"Bar","Time":null}`}, + + // Recursive dict + {NewDict().Set("Foo", + NewDict().Set("Bar", 2). + Set("Time", time.Unix(3000000000000000, 0))), + `{"Foo":{"Bar":2,"Time":null}}`}, + + // Ensure key order is preserved. + {NewDict().Set("A", 1).Set("B", 2), `{"A":1,"B":2}`}, + {NewDict().Set("B", 1).Set("A", 2), `{"B":1,"A":2}`}, + } + + // Check that serialization decodes to the object. + dictUnserializationTest = []dictSerializationTest{ + // Preserve order of keys on deserialization. + {NewDict().Set("A", int64(1)).Set("B", int64(2)), `{"A":1,"B":2}`}, + {NewDict().Set("B", int64(1)).Set("A", int64(2)), `{"B":1,"A":2}`}, + + // Handle arrays, ints floats and bools + {NewDict().Set("B", int64(1)).Set("A", int64(2)), `{"B":1,"A":2}`}, + {NewDict().Set("B", float64(1)).Set("A", int64(2)), `{"B":1.0,"A":2}`}, + {NewDict().Set("B", []interface{}{int64(1)}).Set("A", int64(2)), `{"B":[1],"A":2}`}, + {NewDict().Set("B", true).Set("A", int64(2)), `{"B":true,"A":2}`}, + {NewDict().Set("B", nil).Set("A", int64(2)), `{"B":null,"A":2}`}, + + // Embedded dicts decode into ordered dicts. + {NewDict(). + Set("B", NewDict().Set("Zoo", "X").Set("Baz", "Y")). + Set("A", "Z"), `{"B":{"Zoo":"X","Baz":"Y"},"A":"Z"}`}, + } +) + +func TestDictSerialization(t *testing.T) { + for _, test := range dictSerializationTests { + serialized, err := json.Marshal(test.dict) + if err != nil { + t.Fatalf("Failed to serialize %v: %v", repr.String(test.dict), err) + } + + assert.Equal(t, test.serialized, string(serialized)) + } +} + +func TestDictDeserialization(t *testing.T) { + for _, test := range dictUnserializationTest { + value := NewDict() + + err := json.Unmarshal([]byte(test.serialized), value) + if err != nil { + t.Fatalf("Failed to serialize %v: %v", test.serialized, err) + } + // Make sure the keys are the same. + assert.Equal(t, test.dict.Keys(), value.Keys()) + assert.Equal(t, test.dict.ToDict(), value.ToDict()) + + assert.True(t, reflect.DeepEqual(test.dict, value)) + } +} + +func TestOrder(t *testing.T) { + test := NewDict(). + Set("A", 1). + Set("B", 2) + + assert.Equal(t, []string{"A", "B"}, test.Keys()) + + test = NewDict(). + Set("B", 1). + Set("A", 2) + + assert.Equal(t, []string{"B", "A"}, test.Keys()) +} + +func TestCaseInsensitive(t *testing.T) { + test := NewDict().SetCaseInsensitive() + + test.Set("FOO", 1) + + value, pres := test.Get("foo") + assert.True(t, pres) + assert.Equal(t, 1, value) + + test = NewDict().Set("FOO", 1) + value, pres = test.Get("foo") + assert.False(t, pres) +} From c16e1910ce18d3b01d2f1b2920716dbcf90b826f Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Sun, 3 Nov 2019 10:48:26 +1000 Subject: [PATCH 03/40] Changed package name --- ordereddict.go | 2 +- ordereddict_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ordereddict.go b/ordereddict.go index 265cf15..46aab2f 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -1,4 +1,4 @@ -package vfilter +package ordereddict import ( "bytes" diff --git a/ordereddict_test.go b/ordereddict_test.go index d7f8c4e..d4e1e02 100644 --- a/ordereddict_test.go +++ b/ordereddict_test.go @@ -1,4 +1,4 @@ -package vfilter +package ordereddict import ( "encoding/json" From 3b5a5f6957d4eca2f61390941c271a36bde45323 Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Sun, 3 Nov 2019 11:10:20 +1000 Subject: [PATCH 04/40] Properly escape key values. --- ordereddict.go | 13 ++++++++----- ordereddict_test.go | 3 +++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/ordereddict.go b/ordereddict.go index 46aab2f..bab2ef2 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -318,18 +318,21 @@ func (self Dict) MarshalJSON() ([]byte, error) { for _, k := range self.keys { // add key - kEscaped := strings.Replace(k, `"`, `\"`, -1) - result = result + `"` + kEscaped + `":` + kEscaped, err := json.Marshal(k) + if err != nil { + continue + } + + result += string(kEscaped) + ":" // add value v := self.store[k] vBytes, err := json.Marshal(v) - if err == nil { - result = result + string(vBytes) + "," + result += string(vBytes) + "," } else { - result = result + "null," + result += "null," } } if len(self.keys) > 0 { diff --git a/ordereddict_test.go b/ordereddict_test.go index d4e1e02..fbaccf7 100644 --- a/ordereddict_test.go +++ b/ordereddict_test.go @@ -35,6 +35,9 @@ var ( // Ensure key order is preserved. {NewDict().Set("A", 1).Set("B", 2), `{"A":1,"B":2}`}, {NewDict().Set("B", 1).Set("A", 2), `{"B":1,"A":2}`}, + + // Serialize with quotes + {NewDict().Set("foo\\'s quote", 1), `{"foo\\'s quote":1}`}, } // Check that serialization decodes to the object. From 97c468e5e40365da2fc937cc0c2b7e3e260615ad Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Wed, 6 Nov 2019 12:09:01 +1000 Subject: [PATCH 05/40] Added some useful utilities. (#1) --- utils.go | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 utils.go diff --git a/utils.go b/utils.go new file mode 100644 index 0000000..6f0987d --- /dev/null +++ b/utils.go @@ -0,0 +1,75 @@ +package ordereddict + +import "strings" + +func GetString(event_map *Dict, members string) (string, bool) { + value, pres := GetAny(event_map, members) + if pres { + switch t := value.(type) { + case string: + return t, true + case *string: + return *t, true + } + } + + return "", false +} + +func GetMap(event_map *Dict, members string) (*Dict, bool) { + value, pres := GetAny(event_map, members) + if pres { + switch t := value.(type) { + case *Dict: + return t, true + } + } + return nil, false +} + +func GetAny(event_map *Dict, members string) (interface{}, bool) { + var value interface{} = event_map + var pres bool + + for _, member := range strings.Split(members, ".") { + if event_map == nil { + return nil, false + } + + value, pres = event_map.Get(member) + if !pres { + return nil, false + } + event_map, pres = value.(*Dict) + } + + return value, true +} + +func GetInt(event_map *Dict, members string) (int, bool) { + value, pres := GetAny(event_map, members) + if pres { + switch t := value.(type) { + case int: + return t, true + case uint8: + return int(t), true + case uint16: + return int(t), true + case uint32: + return int(t), true + case uint64: + return int(t), true + case int8: + return int(t), true + case int16: + return int(t), true + case int32: + return int(t), true + case int64: + return int(t), true + } + } + + return 0, false +} From 4194f9d00a6ffd8113056a890c4f9f1881c0709e Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Wed, 27 Nov 2019 00:34:33 +1000 Subject: [PATCH 06/40] Initial work on the parser. --- LICENSE | 201 ++++++++ catalog.go | 123 +++++ context.go | 32 ++ conversion.spec.yaml | 26 ++ ese_gen.go | 1043 ++++++++++++++++++++++++++++++++++++++++++ ese_profile.json | 195 ++++++++ pages.go | 249 ++++++++++ reader.go | 20 + 8 files changed, 1889 insertions(+) create mode 100644 LICENSE create mode 100644 catalog.go create mode 100644 context.go create mode 100644 conversion.spec.yaml create mode 100644 ese_gen.go create mode 100644 ese_profile.json create mode 100644 pages.go create mode 100644 reader.go diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..269653d --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 velocidex + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/catalog.go b/catalog.go new file mode 100644 index 0000000..53dd871 --- /dev/null +++ b/catalog.go @@ -0,0 +1,123 @@ +package parser + +import ( + "errors" + "fmt" + + "github.com/Velocidex/ordereddict" + "github.com/davecgh/go-spew/spew" +) + +const ( + CATALOG_PAGE_NUMBER = 4 +) + +type Table struct { + Header *CATALOG_TYPE_TABLE + FatherDataPageNumber uint32 + Name string + Columns *ordereddict.Dict + Indexes *ordereddict.Dict + LongValues *ordereddict.Dict +} + +type Catalog struct { + ctx *ESEContext + + Tables *ordereddict.Dict + + currentTable *Table +} + +func parseItemName(dd_header *ESENT_DATA_DEFINITION_HEADER) string { + last_variable_data_type := int64(dd_header.LastVariableDataType()) + numEntries := last_variable_data_type + + if last_variable_data_type > 127 { + numEntries = last_variable_data_type - 127 + } + + itemLen := ParseUint16(dd_header.Reader, + dd_header.Offset+int64(dd_header.VariableSizeOffset())) + + return ParseString(dd_header.Reader, + dd_header.Offset+int64(dd_header.VariableSizeOffset())+ + 2*numEntries, int64(itemLen)) +} + +func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) { + leaf_entry := self.ctx.Profile.ESENT_LEAF_ENTRY(value.Reader(), 0) + dd_header := self.ctx.Profile.ESENT_DATA_DEFINITION_HEADER( + leaf_entry.Reader, leaf_entry.EntryData(value)) + + itemName := parseItemName(dd_header) + catalog := dd_header.Catalog() + + switch catalog.Type().Name { + case "CATALOG_TYPE_TABLE": + table := &Table{ + Header: catalog.Table(), + Name: itemName, + FatherDataPageNumber: catalog.FDPId(), + Columns: ordereddict.NewDict(), + Indexes: ordereddict.NewDict(), + LongValues: ordereddict.NewDict()} + self.currentTable = table + self.Tables.Set(itemName, table) + + case "CATALOG_TYPE_COLUMN": + self.currentTable.Columns.Set(itemName, catalog.Column()) + + case "CATALOG_TYPE_INDEX": + self.currentTable.Indexes.Set(itemName, catalog.Index()) + case "CATALOG_TYPE_LONG_VALUE": + + } +} + +func (self *Catalog) DumpTable(name string) error { + table_any, pres := self.Tables.Get(name) + if !pres { + return errors.New("Table not found") + } + + table := table_any.(*Table) + WalkPages(self.ctx, int64(table.FatherDataPageNumber), self._walkTableContents) + + return nil +} + +func (self *Catalog) _walkTableContents(header *PageHeader, id int64, value *Value) { + leaf_entry := self.ctx.Profile.ESENT_LEAF_ENTRY(value.Reader(), 0) + fmt.Printf("Leaf %v\n", leaf_entry.DebugString()) + spew.Dump(value.Buffer) +} + +func (self *Catalog) Dump() { + for _, name := range self.Tables.Keys() { + table_any, _ := self.Tables.Get(name) + table := table_any.(*Table) + + space := " " + fmt.Printf("[%v]:\n%sColumns\n", table.Name, space) + for idx, column := range table.Columns.Keys() { + column_header_any, _ := table.Columns.Get(column) + column_header := column_header_any.(*CATALOG_TYPE_COLUMN) + fmt.Printf("%s%s%-5d%-30v%v\n", space, space, idx, + column, column_header.ColumnType().Name) + } + + fmt.Printf("%sIndexes\n", space) + for _, index := range table.Indexes.Keys() { + fmt.Printf("%s%s%v:\n", space, space, index) + } + fmt.Printf("\n") + } +} + +func ReadCatalog(ctx *ESEContext) *Catalog { + result := &Catalog{ctx: ctx, Tables: ordereddict.NewDict()} + + WalkPages(ctx, CATALOG_PAGE_NUMBER, result.__addItem) + return result +} diff --git a/context.go b/context.go new file mode 100644 index 0000000..3e46b77 --- /dev/null +++ b/context.go @@ -0,0 +1,32 @@ +package parser + +import ( + "io" +) + +type ESEContext struct { + Reader io.ReaderAt + FileSize int64 + Profile *ESEProfile + PageSize int64 + Header *FileHeader +} + +func NewESEContext(reader io.ReaderAt, filesize int64) (*ESEContext, error) { + result := &ESEContext{ + Profile: NewESEProfile(), + Reader: reader, + } + + // TODO error check. + result.Header = result.Profile.FileHeader(reader, 0) + result.PageSize = int64(result.Header.PageSize()) + + return result, nil +} + +func (self *ESEContext) GetPage(id int64) *PageHeader { + // First file page is file header, second page is backup of file header. + result := self.Profile.PageHeader(self.Reader, (id+1)*self.PageSize) + return result +} diff --git a/conversion.spec.yaml b/conversion.spec.yaml new file mode 100644 index 0000000..441f3c7 --- /dev/null +++ b/conversion.spec.yaml @@ -0,0 +1,26 @@ +Module: parser +Profile: ESEProfile +Filename: ese_profile.json +GenerateDebugString: true +Structs: + - FileHeader + - DBTime + - JET_LOGTIME + - JET_SIGNATURE + - PageHeader + - Tag + - ESENT_ROOT_HEADER + - ESENT_BRANCH_HEADER + - ESENT_SPACE_TREE_HEADER + - ESENT_LEAF_HEADER + - ESENT_SPACE_TREE_ENTRY + - ESENT_INDEX_ENTRY + - ESENT_LEAF_ENTRY + - ESENT_BRANCH_ENTRY + - CATALOG_TYPE_TABLE + - CATALOG_TYPE_TABLE + - CATALOG_TYPE_COLUMN + - CATALOG_TYPE_INDEX + - CATALOG_TYPE_LONG_VALUE + - ESENT_DATA_DEFINITION_HEADER + - ESENT_CATALOG_DATA_DEFINITION_ENTRY diff --git a/ese_gen.go b/ese_gen.go new file mode 100644 index 0000000..3f489c3 --- /dev/null +++ b/ese_gen.go @@ -0,0 +1,1043 @@ + +package parser + +// Autogenerated code from ese_profile.json. Do not edit. + +import ( + "encoding/binary" + "fmt" + "bytes" + "io" + "sort" + "strings" + "unicode/utf16" + "unicode/utf8" +) + +var ( + // Depending on autogenerated code we may use this. Add a reference + // to shut the compiler up. + _ = bytes.MinRead + _ = fmt.Sprintf + _ = utf16.Decode + _ = binary.LittleEndian + _ = utf8.RuneError + _ = sort.Strings + _ = strings.Join + _ = io.Copy +) + +func indent(text string) string { + result := []string{} + lines := strings.Split(text,"\n") + for _, line := range lines { + result = append(result, " " + line) + } + return strings.Join(result, "\n") +} + + +type ESEProfile struct { + Off_CATALOG_TYPE_COLUMN_ColumnType int64 + Off_CATALOG_TYPE_COLUMN_SpaceUsage int64 + Off_CATALOG_TYPE_COLUMN_ColumnFlags int64 + Off_CATALOG_TYPE_COLUMN_CodePage int64 + Off_CATALOG_TYPE_INDEX_FatherDataPageNumber int64 + Off_CATALOG_TYPE_INDEX_SpaceUsage int64 + Off_CATALOG_TYPE_INDEX_IndexFlags int64 + Off_CATALOG_TYPE_INDEX_Locale int64 + Off_CATALOG_TYPE_LONG_VALUE_FatherDataPageNumber int64 + Off_CATALOG_TYPE_LONG_VALUE_SpaceUsage int64 + Off_CATALOG_TYPE_LONG_VALUE_LVFlags int64 + Off_CATALOG_TYPE_LONG_VALUE_InitialNumberOfPages int64 + Off_CATALOG_TYPE_TABLE_FatherDataPageNumber int64 + Off_CATALOG_TYPE_TABLE_SpaceUsage int64 + Off_DBTime_Hours int64 + Off_DBTime_Min int64 + Off_DBTime_Sec int64 + Off_ESENT_BRANCH_ENTRY_LocalPageKeySize int64 + Off_ESENT_BRANCH_HEADER_CommonPageKey int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_FDPId int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Type int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Identifier int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Column int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Table int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Index int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_LongValue int64 + Off_ESENT_DATA_DEFINITION_HEADER_LastFixedSize int64 + Off_ESENT_DATA_DEFINITION_HEADER_LastVariableDataType int64 + Off_ESENT_DATA_DEFINITION_HEADER_VariableSizeOffset int64 + Off_ESENT_DATA_DEFINITION_HEADER_Catalog int64 + Off_ESENT_INDEX_ENTRY_RecordPageKey int64 + Off_ESENT_LEAF_ENTRY_CommonPageKeySize int64 + Off_ESENT_LEAF_ENTRY_LocalPageKeySize int64 + Off_ESENT_LEAF_HEADER_CommonPageKey int64 + Off_ESENT_ROOT_HEADER_InitialNumberOfPages int64 + Off_ESENT_ROOT_HEADER_ParentFDP int64 + Off_ESENT_ROOT_HEADER_ExtentSpace int64 + Off_ESENT_ROOT_HEADER_SpaceTreePageNumber int64 + Off_ESENT_SPACE_TREE_ENTRY_PageKeySize int64 + Off_ESENT_SPACE_TREE_ENTRY_LastPageNumber int64 + Off_ESENT_SPACE_TREE_ENTRY_NumberOfPages int64 + Off_FileHeader_Magic int64 + Off_FileHeader_FormatVersion int64 + Off_FileHeader_FormatRevision int64 + Off_FileHeader_FileType int64 + Off_FileHeader_DataBaseTime int64 + Off_FileHeader_Signature int64 + Off_FileHeader_PageSize int64 + Off_JET_LOGTIME_Sec int64 + Off_JET_LOGTIME_Min int64 + Off_JET_LOGTIME_Hours int64 + Off_JET_LOGTIME_Days int64 + Off_JET_LOGTIME_Month int64 + Off_JET_LOGTIME_Year int64 + Off_JET_SIGNATURE_Creation int64 + Off_JET_SIGNATURE_CreatorMachine int64 + Off_PageHeader_LastModified int64 + Off_PageHeader_PreviousPage int64 + Off_PageHeader_NextPage int64 + Off_PageHeader_FatherPage int64 + Off_PageHeader_AvailableDataSize int64 + Off_PageHeader_AvailableDataOffset int64 + Off_PageHeader_AvailablePageTag int64 + Off_PageHeader_Flags int64 + Off_Tag_ValueSize int64 + Off_Tag_Flags int64 + Off_Tag_ValueOffset int64 +} + +func NewESEProfile() *ESEProfile { + // Specific offsets can be tweaked to cater for slight version mismatches. + self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,4,0,0,2,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,1,2,3,4,5,4,12,8,16,20,24,28,32,34,36,0,2,2} + return self +} + +func (self *ESEProfile) CATALOG_TYPE_COLUMN(reader io.ReaderAt, offset int64) *CATALOG_TYPE_COLUMN { + return &CATALOG_TYPE_COLUMN{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) CATALOG_TYPE_INDEX(reader io.ReaderAt, offset int64) *CATALOG_TYPE_INDEX { + return &CATALOG_TYPE_INDEX{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) CATALOG_TYPE_LONG_VALUE(reader io.ReaderAt, offset int64) *CATALOG_TYPE_LONG_VALUE { + return &CATALOG_TYPE_LONG_VALUE{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) CATALOG_TYPE_TABLE(reader io.ReaderAt, offset int64) *CATALOG_TYPE_TABLE { + return &CATALOG_TYPE_TABLE{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) DBTime(reader io.ReaderAt, offset int64) *DBTime { + return &DBTime{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_BRANCH_ENTRY(reader io.ReaderAt, offset int64) *ESENT_BRANCH_ENTRY { + return &ESENT_BRANCH_ENTRY{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_BRANCH_HEADER(reader io.ReaderAt, offset int64) *ESENT_BRANCH_HEADER { + return &ESENT_BRANCH_HEADER{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_CATALOG_DATA_DEFINITION_ENTRY(reader io.ReaderAt, offset int64) *ESENT_CATALOG_DATA_DEFINITION_ENTRY { + return &ESENT_CATALOG_DATA_DEFINITION_ENTRY{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_DATA_DEFINITION_HEADER(reader io.ReaderAt, offset int64) *ESENT_DATA_DEFINITION_HEADER { + return &ESENT_DATA_DEFINITION_HEADER{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_INDEX_ENTRY(reader io.ReaderAt, offset int64) *ESENT_INDEX_ENTRY { + return &ESENT_INDEX_ENTRY{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_LEAF_ENTRY(reader io.ReaderAt, offset int64) *ESENT_LEAF_ENTRY { + return &ESENT_LEAF_ENTRY{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_LEAF_HEADER(reader io.ReaderAt, offset int64) *ESENT_LEAF_HEADER { + return &ESENT_LEAF_HEADER{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_ROOT_HEADER(reader io.ReaderAt, offset int64) *ESENT_ROOT_HEADER { + return &ESENT_ROOT_HEADER{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_SPACE_TREE_ENTRY(reader io.ReaderAt, offset int64) *ESENT_SPACE_TREE_ENTRY { + return &ESENT_SPACE_TREE_ENTRY{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_SPACE_TREE_HEADER(reader io.ReaderAt, offset int64) *ESENT_SPACE_TREE_HEADER { + return &ESENT_SPACE_TREE_HEADER{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) FileHeader(reader io.ReaderAt, offset int64) *FileHeader { + return &FileHeader{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) JET_LOGTIME(reader io.ReaderAt, offset int64) *JET_LOGTIME { + return &JET_LOGTIME{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) JET_SIGNATURE(reader io.ReaderAt, offset int64) *JET_SIGNATURE { + return &JET_SIGNATURE{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) PageHeader(reader io.ReaderAt, offset int64) *PageHeader { + return &PageHeader{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) Tag(reader io.ReaderAt, offset int64) *Tag { + return &Tag{Reader: reader, Offset: offset, Profile: self} +} + + +type CATALOG_TYPE_COLUMN struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *CATALOG_TYPE_COLUMN) Size() int { + return 0 +} + +func (self *CATALOG_TYPE_COLUMN) ColumnType() *Enumeration { + value := ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_ColumnType + self.Offset) + name := "Unknown" + switch value { + + case 0: + name = "NULL" + + case 1: + name = "Boolean" + + case 2: + name = "Signed byte" + + case 3: + name = "Signed short" + + case 4: + name = "Signed long" + + case 5: + name = "Currency" + + case 6: + name = "Single precision FP" + + case 7: + name = "Double precision FP" + + case 8: + name = "DateTime" + + case 9: + name = "Binary" + + case 10: + name = "Text" + + case 11: + name = "Long Binary" + + case 12: + name = "Long Text" + + case 13: + name = "Obsolete" + + case 14: + name = "Unsigned long" + + case 15: + name = "Long long" + + case 16: + name = "GUID" + + case 17: + name = "Unsigned short" + + case 18: + name = "Max" +} + return &Enumeration{Value: uint64(value), Name: name} +} + + +func (self *CATALOG_TYPE_COLUMN) SpaceUsage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_SpaceUsage + self.Offset) +} + +func (self *CATALOG_TYPE_COLUMN) ColumnFlags() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_ColumnFlags + self.Offset) +} + +func (self *CATALOG_TYPE_COLUMN) CodePage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_CodePage + self.Offset) +} +func (self *CATALOG_TYPE_COLUMN) DebugString() string { + result := fmt.Sprintf("struct CATALOG_TYPE_COLUMN @ %#x:\n", self.Offset) + result += fmt.Sprintf(" ColumnType: %v\n", self.ColumnType().DebugString()) + result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) + result += fmt.Sprintf(" ColumnFlags: %#0x\n", self.ColumnFlags()) + result += fmt.Sprintf(" CodePage: %#0x\n", self.CodePage()) + return result +} + +type CATALOG_TYPE_INDEX struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *CATALOG_TYPE_INDEX) Size() int { + return 0 +} + +func (self *CATALOG_TYPE_INDEX) FatherDataPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_FatherDataPageNumber + self.Offset) +} + +func (self *CATALOG_TYPE_INDEX) SpaceUsage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_SpaceUsage + self.Offset) +} + +func (self *CATALOG_TYPE_INDEX) IndexFlags() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_IndexFlags + self.Offset) +} + +func (self *CATALOG_TYPE_INDEX) Locale() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_Locale + self.Offset) +} +func (self *CATALOG_TYPE_INDEX) DebugString() string { + result := fmt.Sprintf("struct CATALOG_TYPE_INDEX @ %#x:\n", self.Offset) + result += fmt.Sprintf(" FatherDataPageNumber: %#0x\n", self.FatherDataPageNumber()) + result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) + result += fmt.Sprintf(" IndexFlags: %#0x\n", self.IndexFlags()) + result += fmt.Sprintf(" Locale: %#0x\n", self.Locale()) + return result +} + +type CATALOG_TYPE_LONG_VALUE struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *CATALOG_TYPE_LONG_VALUE) Size() int { + return 0 +} + +func (self *CATALOG_TYPE_LONG_VALUE) FatherDataPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_FatherDataPageNumber + self.Offset) +} + +func (self *CATALOG_TYPE_LONG_VALUE) SpaceUsage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_SpaceUsage + self.Offset) +} + +func (self *CATALOG_TYPE_LONG_VALUE) LVFlags() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_LVFlags + self.Offset) +} + +func (self *CATALOG_TYPE_LONG_VALUE) InitialNumberOfPages() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_InitialNumberOfPages + self.Offset) +} +func (self *CATALOG_TYPE_LONG_VALUE) DebugString() string { + result := fmt.Sprintf("struct CATALOG_TYPE_LONG_VALUE @ %#x:\n", self.Offset) + result += fmt.Sprintf(" FatherDataPageNumber: %#0x\n", self.FatherDataPageNumber()) + result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) + result += fmt.Sprintf(" LVFlags: %#0x\n", self.LVFlags()) + result += fmt.Sprintf(" InitialNumberOfPages: %#0x\n", self.InitialNumberOfPages()) + return result +} + +type CATALOG_TYPE_TABLE struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *CATALOG_TYPE_TABLE) Size() int { + return 0 +} + +func (self *CATALOG_TYPE_TABLE) FatherDataPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_TABLE_FatherDataPageNumber + self.Offset) +} + +func (self *CATALOG_TYPE_TABLE) SpaceUsage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_TABLE_SpaceUsage + self.Offset) +} +func (self *CATALOG_TYPE_TABLE) DebugString() string { + result := fmt.Sprintf("struct CATALOG_TYPE_TABLE @ %#x:\n", self.Offset) + result += fmt.Sprintf(" FatherDataPageNumber: %#0x\n", self.FatherDataPageNumber()) + result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) + return result +} + +type DBTime struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *DBTime) Size() int { + return 8 +} + +func (self *DBTime) Hours() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_DBTime_Hours + self.Offset) +} + +func (self *DBTime) Min() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_DBTime_Min + self.Offset) +} + +func (self *DBTime) Sec() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_DBTime_Sec + self.Offset) +} +func (self *DBTime) DebugString() string { + result := fmt.Sprintf("struct DBTime @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Hours: %#0x\n", self.Hours()) + result += fmt.Sprintf(" Min: %#0x\n", self.Min()) + result += fmt.Sprintf(" Sec: %#0x\n", self.Sec()) + return result +} + +type ESENT_BRANCH_ENTRY struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_BRANCH_ENTRY) Size() int { + return 16 +} + +func (self *ESENT_BRANCH_ENTRY) LocalPageKeySize() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_ESENT_BRANCH_ENTRY_LocalPageKeySize + self.Offset) +} +func (self *ESENT_BRANCH_ENTRY) DebugString() string { + result := fmt.Sprintf("struct ESENT_BRANCH_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" LocalPageKeySize: %#0x\n", self.LocalPageKeySize()) + return result +} + +type ESENT_BRANCH_HEADER struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_BRANCH_HEADER) Size() int { + return 16 +} + + +func (self *ESENT_BRANCH_HEADER) CommonPageKey() string { + return ParseTerminatedString(self.Reader, self.Profile.Off_ESENT_BRANCH_HEADER_CommonPageKey + self.Offset) +} +func (self *ESENT_BRANCH_HEADER) DebugString() string { + result := fmt.Sprintf("struct ESENT_BRANCH_HEADER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" CommonPageKey: %v\n", string(self.CommonPageKey())) + return result +} + +type ESENT_CATALOG_DATA_DEFINITION_ENTRY struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Size() int { + return 0 +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) FDPId() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_FDPId + self.Offset) +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Type() *Enumeration { + value := ParseUint16(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Type + self.Offset) + name := "Unknown" + switch value { + + case 1: + name = "CATALOG_TYPE_TABLE" + + case 2: + name = "CATALOG_TYPE_COLUMN" + + case 3: + name = "CATALOG_TYPE_INDEX" + + case 4: + name = "CATALOG_TYPE_LONG_VALUE" +} + return &Enumeration{Value: uint64(value), Name: name} +} + + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Identifier() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Identifier + self.Offset) +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Column() *CATALOG_TYPE_COLUMN { + return self.Profile.CATALOG_TYPE_COLUMN(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Column + self.Offset) +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Table() *CATALOG_TYPE_TABLE { + return self.Profile.CATALOG_TYPE_TABLE(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Table + self.Offset) +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Index() *CATALOG_TYPE_INDEX { + return self.Profile.CATALOG_TYPE_INDEX(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Index + self.Offset) +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) LongValue() *CATALOG_TYPE_LONG_VALUE { + return self.Profile.CATALOG_TYPE_LONG_VALUE(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_LongValue + self.Offset) +} +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) DebugString() string { + result := fmt.Sprintf("struct ESENT_CATALOG_DATA_DEFINITION_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" FDPId: %#0x\n", self.FDPId()) + result += fmt.Sprintf(" Type: %v\n", self.Type().DebugString()) + result += fmt.Sprintf(" Identifier: %#0x\n", self.Identifier()) + result += fmt.Sprintf(" Column: {\n%v}\n", indent(self.Column().DebugString())) + result += fmt.Sprintf(" Table: {\n%v}\n", indent(self.Table().DebugString())) + result += fmt.Sprintf(" Index: {\n%v}\n", indent(self.Index().DebugString())) + result += fmt.Sprintf(" LongValue: {\n%v}\n", indent(self.LongValue().DebugString())) + return result +} + +type ESENT_DATA_DEFINITION_HEADER struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_DATA_DEFINITION_HEADER) Size() int { + return 0 +} + +func (self *ESENT_DATA_DEFINITION_HEADER) LastFixedSize() int8 { + return ParseInt8(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_LastFixedSize + self.Offset) +} + +func (self *ESENT_DATA_DEFINITION_HEADER) LastVariableDataType() byte { + return ParseUint8(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_LastVariableDataType + self.Offset) +} + +func (self *ESENT_DATA_DEFINITION_HEADER) VariableSizeOffset() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_VariableSizeOffset + self.Offset) +} + +func (self *ESENT_DATA_DEFINITION_HEADER) Catalog() *ESENT_CATALOG_DATA_DEFINITION_ENTRY { + return self.Profile.ESENT_CATALOG_DATA_DEFINITION_ENTRY(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_Catalog + self.Offset) +} +func (self *ESENT_DATA_DEFINITION_HEADER) DebugString() string { + result := fmt.Sprintf("struct ESENT_DATA_DEFINITION_HEADER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" LastFixedSize: %#0x\n", self.LastFixedSize()) + result += fmt.Sprintf(" LastVariableDataType: %#0x\n", self.LastVariableDataType()) + result += fmt.Sprintf(" VariableSizeOffset: %#0x\n", self.VariableSizeOffset()) + result += fmt.Sprintf(" Catalog: {\n%v}\n", indent(self.Catalog().DebugString())) + return result +} + +type ESENT_INDEX_ENTRY struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_INDEX_ENTRY) Size() int { + return 16 +} + + +func (self *ESENT_INDEX_ENTRY) RecordPageKey() string { + return ParseTerminatedString(self.Reader, self.Profile.Off_ESENT_INDEX_ENTRY_RecordPageKey + self.Offset) +} +func (self *ESENT_INDEX_ENTRY) DebugString() string { + result := fmt.Sprintf("struct ESENT_INDEX_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" RecordPageKey: %v\n", string(self.RecordPageKey())) + return result +} + +type ESENT_LEAF_ENTRY struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_LEAF_ENTRY) Size() int { + return 16 +} + +func (self *ESENT_LEAF_ENTRY) CommonPageKeySize() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_ESENT_LEAF_ENTRY_CommonPageKeySize + self.Offset) +} + +func (self *ESENT_LEAF_ENTRY) LocalPageKeySize() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_ESENT_LEAF_ENTRY_LocalPageKeySize + self.Offset) +} +func (self *ESENT_LEAF_ENTRY) DebugString() string { + result := fmt.Sprintf("struct ESENT_LEAF_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" CommonPageKeySize: %#0x\n", self.CommonPageKeySize()) + result += fmt.Sprintf(" LocalPageKeySize: %#0x\n", self.LocalPageKeySize()) + return result +} + +type ESENT_LEAF_HEADER struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_LEAF_HEADER) Size() int { + return 16 +} + + +func (self *ESENT_LEAF_HEADER) CommonPageKey() string { + return ParseTerminatedString(self.Reader, self.Profile.Off_ESENT_LEAF_HEADER_CommonPageKey + self.Offset) +} +func (self *ESENT_LEAF_HEADER) DebugString() string { + result := fmt.Sprintf("struct ESENT_LEAF_HEADER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" CommonPageKey: %v\n", string(self.CommonPageKey())) + return result +} + +type ESENT_ROOT_HEADER struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_ROOT_HEADER) Size() int { + return 16 +} + +func (self *ESENT_ROOT_HEADER) InitialNumberOfPages() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_InitialNumberOfPages + self.Offset) +} + +func (self *ESENT_ROOT_HEADER) ParentFDP() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_ParentFDP + self.Offset) +} + +func (self *ESENT_ROOT_HEADER) ExtentSpace() *Enumeration { + value := ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_ExtentSpace + self.Offset) + name := "Unknown" + switch value { + + case 0: + name = "Single" + + case 1: + name = "Multiple" +} + return &Enumeration{Value: uint64(value), Name: name} +} + + +func (self *ESENT_ROOT_HEADER) SpaceTreePageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_SpaceTreePageNumber + self.Offset) +} +func (self *ESENT_ROOT_HEADER) DebugString() string { + result := fmt.Sprintf("struct ESENT_ROOT_HEADER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" InitialNumberOfPages: %#0x\n", self.InitialNumberOfPages()) + result += fmt.Sprintf(" ParentFDP: %#0x\n", self.ParentFDP()) + result += fmt.Sprintf(" ExtentSpace: %v\n", self.ExtentSpace().DebugString()) + result += fmt.Sprintf(" SpaceTreePageNumber: %#0x\n", self.SpaceTreePageNumber()) + return result +} + +type ESENT_SPACE_TREE_ENTRY struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_SPACE_TREE_ENTRY) Size() int { + return 16 +} + +func (self *ESENT_SPACE_TREE_ENTRY) PageKeySize() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_ESENT_SPACE_TREE_ENTRY_PageKeySize + self.Offset) +} + +func (self *ESENT_SPACE_TREE_ENTRY) LastPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_SPACE_TREE_ENTRY_LastPageNumber + self.Offset) +} + +func (self *ESENT_SPACE_TREE_ENTRY) NumberOfPages() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_SPACE_TREE_ENTRY_NumberOfPages + self.Offset) +} +func (self *ESENT_SPACE_TREE_ENTRY) DebugString() string { + result := fmt.Sprintf("struct ESENT_SPACE_TREE_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" PageKeySize: %#0x\n", self.PageKeySize()) + result += fmt.Sprintf(" LastPageNumber: %#0x\n", self.LastPageNumber()) + result += fmt.Sprintf(" NumberOfPages: %#0x\n", self.NumberOfPages()) + return result +} + +type ESENT_SPACE_TREE_HEADER struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_SPACE_TREE_HEADER) Size() int { + return 16 +} +func (self *ESENT_SPACE_TREE_HEADER) DebugString() string { + result := fmt.Sprintf("struct ESENT_SPACE_TREE_HEADER @ %#x:\n", self.Offset) + return result +} + +type FileHeader struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *FileHeader) Size() int { + return 0 +} + +func (self *FileHeader) Magic() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_FileHeader_Magic + self.Offset) +} + +func (self *FileHeader) FormatVersion() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_FileHeader_FormatVersion + self.Offset) +} + +func (self *FileHeader) FormatRevision() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_FileHeader_FormatRevision + self.Offset) +} + +func (self *FileHeader) FileType() *Enumeration { + value := ParseUint32(self.Reader, self.Profile.Off_FileHeader_FileType + self.Offset) + name := "Unknown" + switch value { + + case 0: + name = "Database" + + case 1: + name = "StreamingFile" +} + return &Enumeration{Value: uint64(value), Name: name} +} + + +func (self *FileHeader) DataBaseTime() *DBTime { + return self.Profile.DBTime(self.Reader, self.Profile.Off_FileHeader_DataBaseTime + self.Offset) +} + +func (self *FileHeader) Signature() *JET_SIGNATURE { + return self.Profile.JET_SIGNATURE(self.Reader, self.Profile.Off_FileHeader_Signature + self.Offset) +} + +func (self *FileHeader) PageSize() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_FileHeader_PageSize + self.Offset) +} +func (self *FileHeader) DebugString() string { + result := fmt.Sprintf("struct FileHeader @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Magic: %#0x\n", self.Magic()) + result += fmt.Sprintf(" FormatVersion: %#0x\n", self.FormatVersion()) + result += fmt.Sprintf(" FormatRevision: %#0x\n", self.FormatRevision()) + result += fmt.Sprintf(" FileType: %v\n", self.FileType().DebugString()) + result += fmt.Sprintf(" DataBaseTime: {\n%v}\n", indent(self.DataBaseTime().DebugString())) + result += fmt.Sprintf(" Signature: {\n%v}\n", indent(self.Signature().DebugString())) + result += fmt.Sprintf(" PageSize: %#0x\n", self.PageSize()) + return result +} + +type JET_LOGTIME struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *JET_LOGTIME) Size() int { + return 8 +} + +func (self *JET_LOGTIME) Sec() byte { + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Sec + self.Offset) +} + +func (self *JET_LOGTIME) Min() byte { + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Min + self.Offset) +} + +func (self *JET_LOGTIME) Hours() byte { + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Hours + self.Offset) +} + +func (self *JET_LOGTIME) Days() byte { + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Days + self.Offset) +} + +func (self *JET_LOGTIME) Month() byte { + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Month + self.Offset) +} + +func (self *JET_LOGTIME) Year() byte { + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Year + self.Offset) +} +func (self *JET_LOGTIME) DebugString() string { + result := fmt.Sprintf("struct JET_LOGTIME @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Sec: %#0x\n", self.Sec()) + result += fmt.Sprintf(" Min: %#0x\n", self.Min()) + result += fmt.Sprintf(" Hours: %#0x\n", self.Hours()) + result += fmt.Sprintf(" Days: %#0x\n", self.Days()) + result += fmt.Sprintf(" Month: %#0x\n", self.Month()) + result += fmt.Sprintf(" Year: %#0x\n", self.Year()) + return result +} + +type JET_SIGNATURE struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *JET_SIGNATURE) Size() int { + return 28 +} + +func (self *JET_SIGNATURE) Creation() *JET_LOGTIME { + return self.Profile.JET_LOGTIME(self.Reader, self.Profile.Off_JET_SIGNATURE_Creation + self.Offset) +} + + +func (self *JET_SIGNATURE) CreatorMachine() string { + return ParseTerminatedString(self.Reader, self.Profile.Off_JET_SIGNATURE_CreatorMachine + self.Offset) +} +func (self *JET_SIGNATURE) DebugString() string { + result := fmt.Sprintf("struct JET_SIGNATURE @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Creation: {\n%v}\n", indent(self.Creation().DebugString())) + result += fmt.Sprintf(" CreatorMachine: %v\n", string(self.CreatorMachine())) + return result +} + +type PageHeader struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *PageHeader) Size() int { + return 0 +} + +func (self *PageHeader) LastModified() *DBTime { + return self.Profile.DBTime(self.Reader, self.Profile.Off_PageHeader_LastModified + self.Offset) +} + +func (self *PageHeader) PreviousPage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_PageHeader_PreviousPage + self.Offset) +} + +func (self *PageHeader) NextPage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_PageHeader_NextPage + self.Offset) +} + +func (self *PageHeader) FatherPage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_PageHeader_FatherPage + self.Offset) +} + +func (self *PageHeader) AvailableDataSize() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_PageHeader_AvailableDataSize + self.Offset) +} + +func (self *PageHeader) AvailableDataOffset() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_PageHeader_AvailableDataOffset + self.Offset) +} + +func (self *PageHeader) AvailablePageTag() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_PageHeader_AvailablePageTag + self.Offset) +} + +func (self *PageHeader) Flags() *Flags { + value := ParseUint32(self.Reader, self.Profile.Off_PageHeader_Flags + self.Offset) + names := make(map[string]bool) + + + if value & 128 != 0 { + names["Long"] = true + } + + if value & 1 != 0 { + names["Root"] = true + } + + if value & 2 != 0 { + names["Leaf"] = true + } + + if value & 4 != 0 { + names["Parent"] = true + } + + if value & 8 != 0 { + names["Empty"] = true + } + + if value & 32 != 0 { + names["SpaceTree"] = true + } + + if value & 64 != 0 { + names["Index"] = true + } + + return &Flags{Value: uint64(value), Names: names} +} + +func (self *PageHeader) DebugString() string { + result := fmt.Sprintf("struct PageHeader @ %#x:\n", self.Offset) + result += fmt.Sprintf(" LastModified: {\n%v}\n", indent(self.LastModified().DebugString())) + result += fmt.Sprintf(" PreviousPage: %#0x\n", self.PreviousPage()) + result += fmt.Sprintf(" NextPage: %#0x\n", self.NextPage()) + result += fmt.Sprintf(" FatherPage: %#0x\n", self.FatherPage()) + result += fmt.Sprintf(" AvailableDataSize: %#0x\n", self.AvailableDataSize()) + result += fmt.Sprintf(" AvailableDataOffset: %#0x\n", self.AvailableDataOffset()) + result += fmt.Sprintf(" AvailablePageTag: %#0x\n", self.AvailablePageTag()) + result += fmt.Sprintf(" Flags: %v\n", self.Flags().DebugString()) + return result +} + +type Tag struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *Tag) Size() int { + return 8 +} + +func (self *Tag) ValueSize() uint64 { + value := ParseUint16(self.Reader, self.Profile.Off_Tag_ValueSize + self.Offset) + return (uint64(value) & 0x1fff) >> 0x0 +} + +func (self *Tag) Flags() uint64 { + value := ParseUint16(self.Reader, self.Profile.Off_Tag_Flags + self.Offset) + return (uint64(value) & 0x1fffffff) >> 0xd +} + +func (self *Tag) ValueOffset() uint64 { + value := ParseUint16(self.Reader, self.Profile.Off_Tag_ValueOffset + self.Offset) + return (uint64(value) & 0x1fff) >> 0x0 +} +func (self *Tag) DebugString() string { + result := fmt.Sprintf("struct Tag @ %#x:\n", self.Offset) + result += fmt.Sprintf(" ValueSize: %#0x\n", self.ValueSize()) + result += fmt.Sprintf(" Flags: %#0x\n", self.Flags()) + result += fmt.Sprintf(" ValueOffset: %#0x\n", self.ValueOffset()) + return result +} + +type Enumeration struct { + Value uint64 + Name string +} + +func (self Enumeration) DebugString() string { + return fmt.Sprintf("%s (%d)", self.Name, self.Value) +} + + +type Flags struct { + Value uint64 + Names map[string]bool +} + +func (self Flags) DebugString() string { + names := []string{} + for k, _ := range self.Names { + names = append(names, k) + } + + sort.Strings(names) + + return fmt.Sprintf("%d (%s)", self.Value, strings.Join(names, ",")) +} + +func (self Flags) IsSet(flag string) bool { + result, _ := self.Names[flag] + return result +} + + +func ParseInt8(reader io.ReaderAt, offset int64) int8 { + result := make([]byte, 1) + _, err := reader.ReadAt(result, offset) + if err != nil { + return 0 + } + return int8(result[0]) +} + +func ParseUint16(reader io.ReaderAt, offset int64) uint16 { + data := make([]byte, 2) + _, err := reader.ReadAt(data, offset) + if err != nil { + return 0 + } + return binary.LittleEndian.Uint16(data) +} + +func ParseUint32(reader io.ReaderAt, offset int64) uint32 { + data := make([]byte, 4) + _, err := reader.ReadAt(data, offset) + if err != nil { + return 0 + } + return binary.LittleEndian.Uint32(data) +} + +func ParseUint8(reader io.ReaderAt, offset int64) byte { + result := make([]byte, 1) + _, err := reader.ReadAt(result, offset) + if err != nil { + return 0 + } + return result[0] +} + +func ParseTerminatedString(reader io.ReaderAt, offset int64) string { + data := make([]byte, 1024) + n, err := reader.ReadAt(data, offset) + if err != nil && err != io.EOF { + return "" + } + idx := bytes.Index(data[:n], []byte{0}) + if idx < 0 { + idx = n + } + return string(data[0:idx]) +} + +func ParseString(reader io.ReaderAt, offset int64, length int64) string { + data := make([]byte, length) + n, err := reader.ReadAt(data, offset) + if err != nil && err != io.EOF { + return "" + } + return string(data[:n]) +} + + diff --git a/ese_profile.json b/ese_profile.json new file mode 100644 index 0000000..13dd554 --- /dev/null +++ b/ese_profile.json @@ -0,0 +1,195 @@ +{ + "FileHeader": [0, { + "Magic": [4, ["unsigned long"]], + "FormatVersion": [8, ["unsigned long"]], + "FormatRevision": [232, ["unsigned long"]], + "FileType": [12, ["Enumeration", { + "target": "unsigned long", + "choices": { + "0": "Database", + "1": "StreamingFile" + } + }]], + "DataBaseTime": [16, ["DBTime"]], + "Signature": [24, ["JET_SIGNATURE"]], + "PageSize": [236, ["unsigned long"]] + }], + "DBTime": [8, { + "Hours": [0, ["unsigned short"]], + "Min": [2, ["unsigned short"]], + "Sec": [4, ["unsigned short"]] + }], + "JET_SIGNATURE": [28, { + "Creation": [4, ["JET_LOGTIME"]], + "CreatorMachine": [12, ["String"]] + }], + "JET_LOGTIME": [8, { + "Sec": [0, ["unsigned char"]], + "Min": [1, ["unsigned char"]], + "Hours": [2, ["unsigned char"]], + "Days": [3, ["unsigned char"]], + "Month": [4, ["unsigned char"]], + "Year": [5, ["unsigned char"]] + }], + "PageHeader": [0, { + "LastModified": [8, ["DBTime"]], + "PreviousPage": [16, ["unsigned long"]], + "NextPage": [20, ["unsigned long"]], + "FatherPage": [24, ["unsigned long"]], + "AvailableDataSize": [28, ["unsigned short"]], + "AvailableDataOffset": [32, ["unsigned short"]], + "AvailablePageTag": [34, ["unsigned short"]], + "Flags": [36, ["Flags", { + "target": "unsigned long", + "maskmap": { + "Root": 1, + "Leaf": 2, + "Parent": 4, + "Empty": 8, + "SpaceTree": 32, + "Index": 64, + "Long": 128 + } + }]] + }], + "Tag": [8, { + "ValueSize": [0, ["BitField", { + "target": "unsigned short", + "start_bit": 0, + "end_bit": 13 + }]], + "Flags": [2, ["BitField", { + "target": "unsigned short", + "start_bit": 13, + "end_bit": 16 + }]], + "ValueOffset": [2, ["BitField", { + "target": "unsigned short", + "start_bit": 0, + "end_bit": 13 + }]] + }], + "ESENT_ROOT_HEADER": [16, { + "InitialNumberOfPages": [0, ["unsigned long"]], + "ParentFDP": [4, ["unsigned long"]], + "ExtentSpace": [8, ["Enumeration", { + "target": "unsigned long", + "choices": { + "0": "Single", + "1": "Multiple" + } + }]], + "SpaceTreePageNumber": [12, ["unsigned long"]] + }], + + "ESENT_BRANCH_HEADER": [16, { + "CommonPageKey": [0, ["String"]] + }], + + "ESENT_SPACE_TREE_HEADER": [16, { + }], + + "ESENT_LEAF_HEADER": [16, { + "CommonPageKey": [0, ["String"]] + }], + + "ESENT_SPACE_TREE_ENTRY": [16, { + "PageKeySize": [0, ["unsigned short"]], + "LastPageNumber": [0, ["unsigned long"]], + "NumberOfPages": [0, ["unsigned long"]] + }], + + "ESENT_INDEX_ENTRY": [16, { + "RecordPageKey": [0, ["String"]] + }], + + "ESENT_LEAF_ENTRY": [16, { + "CommonPageKeySize": [0, ["unsigned short"]], + "LocalPageKeySize": [2, ["unsigned short"]] + }], + + "ESENT_BRANCH_ENTRY": [16, { + "LocalPageKeySize": [0, ["unsigned short"]] + }], + + "CATALOG_TYPE_TABLE": [0, { + "FatherDataPageNumber": [0, ["unsigned long"]], + "SpaceUsage": [4, ["unsigned long"]], + "Flags": [8, ["unsigned long"]], + "InitialNumberOfPages": [12, ["unsigned long"]] + }], + + "CATALOG_TYPE_TABLE": [0, { + "FatherDataPageNumber": [0, ["unsigned long"]], + "SpaceUsage": [4, ["unsigned long"]] + }], + + "CATALOG_TYPE_COLUMN": [0, { + "ColumnType": [0, ["Enumeration", { + "target": "unsigned long", + "choices": { + "0" : "NULL", + "1" : "Boolean", + "2" : "Signed byte", + "3" : "Signed short", + "4" : "Signed long", + "5" : "Currency", + "6" : "Single precision FP", + "7" : "Double precision FP", + "8" : "DateTime", + "9" : "Binary", + "10" : "Text", + "11" : "Long Binary", + "12" : "Long Text", + "13" : "Obsolete", + "14" : "Unsigned long", + "15" : "Long long", + "16" : "GUID", + "17" : "Unsigned short", + "18" : "Max" + } + }]], + "SpaceUsage": [4, ["unsigned long"]], + "ColumnFlags": [8, ["unsigned long"]], + "CodePage": [12, ["unsigned long"]] + }], + + "CATALOG_TYPE_INDEX": [0, { + "FatherDataPageNumber": [0, ["unsigned long"]], + "SpaceUsage": [4, ["unsigned long"]], + "IndexFlags": [8, ["unsigned long"]], + "Locale": [12, ["unsigned long"]] + }], + + "CATALOG_TYPE_LONG_VALUE": [0, { + "FatherDataPageNumber": [0, ["unsigned long"]], + "SpaceUsage": [4, ["unsigned long"]], + "LVFlags": [8, ["unsigned long"]], + "InitialNumberOfPages": [12, ["unsigned long"]] + }], + + "ESENT_DATA_DEFINITION_HEADER": [0, { + "LastFixedSize": [0, ["char"]], + "LastVariableDataType": [1, ["unsigned char"]], + "VariableSizeOffset": [2, ["unsigned short"]], + "Catalog": [4, ["ESENT_CATALOG_DATA_DEFINITION_ENTRY"]] + }], + + "ESENT_CATALOG_DATA_DEFINITION_ENTRY": [0, { + "FDPId": [0, ["unsigned long"]], + "Type": [4, ["Enumeration", { + "target": "unsigned short", + "choices": { + "1": "CATALOG_TYPE_TABLE", + "2": "CATALOG_TYPE_COLUMN", + "3": "CATALOG_TYPE_INDEX", + "4": "CATALOG_TYPE_LONG_VALUE" + } + }]], + "Identifier": [6, ["unsigned long"]], + "Column": [10, ["CATALOG_TYPE_COLUMN"]], + "Table": [10, ["CATALOG_TYPE_TABLE"]], + "Index": [10, ["CATALOG_TYPE_INDEX"]], + "LongValue": [10, ["CATALOG_TYPE_LONG_VALUE"]] + }] +} diff --git a/pages.go b/pages.go new file mode 100644 index 0000000..02e8e2b --- /dev/null +++ b/pages.go @@ -0,0 +1,249 @@ +package parser + +import ( + "errors" + "fmt" + "io" +) + +const ( + TAG_COMMON = 4 +) + +type Value struct { + Tag *Tag + Buffer []byte +} + +func (self *Value) Reader() io.ReaderAt { + return &BufferReaderAt{self.Buffer} +} + +func GetPageValues(ctx *ESEContext, header *PageHeader) []*Value { + result := []*Value{} + // Tags are written from the end of the page + + offset := ctx.PageSize + header.Offset - 4 + + for tag_count := header.AvailablePageTag(); tag_count > 0; tag_count-- { + tag := ctx.Profile.Tag(ctx.Reader, offset) + value_offset := header.Offset + 40 + int64(tag.ValueOffset()) + + buffer := make([]byte, int(tag.ValueSize())) + ctx.Reader.ReadAt(buffer, value_offset) + + result = append(result, &Value{Tag: tag, Buffer: buffer}) + offset -= 4 + } + + return result +} + +func RootNodeVisitor(ctx *ESEContext, id int64) error { + fmt.Printf("RootNodeVisitor: %v\n", id) + + root_page := ctx.GetPage(id) + if !root_page.Flags().IsSet("Root") { + return errors.New(fmt.Sprintf("ID %v: Not root node", id)) + } + + values := GetPageValues(ctx, root_page) + if len(values) == 0 { + return nil + } + + reader := &BufferReaderAt{values[0].Buffer} + root_header := ctx.Profile.ESENT_ROOT_HEADER(reader, 0) + + if root_header.ExtentSpace().Value > 0 { + space_tree_id := int64(root_header.SpaceTreePageNumber()) + if space_tree_id > 0xff000000 { + return errors.New( + fmt.Sprintf("ID %v: Unsupported extent tree", id)) + } + + if space_tree_id > 0 { + // Left node + err := SpaceNodeVisitor(ctx, space_tree_id) + if err != nil { + return err + } + + // Right node + err = SpaceNodeVisitor(ctx, space_tree_id+1) + if err != nil { + return err + } + } + } + + return nil +} + +func SpaceNodeVisitor(ctx *ESEContext, id int64) error { + fmt.Printf("SpaceNodeVisitor: %v\n", id) + + page := ctx.GetPage(id) + if !page.Flags().IsSet("Root") || + !page.Flags().IsSet("SpaceTree") { + return errors.New(fmt.Sprintf("ID %v: Not SpaceTree node", id)) + } + + if page.NextPage() != 0 { + return errors.New(fmt.Sprintf("ID %v: Next Page not 0", id)) + } + + if page.PreviousPage() != 0 { + return errors.New(fmt.Sprintf("ID %v: Prev Page not 0", id)) + } + + values := GetPageValues(ctx, page) + if len(values) == 0 { + return nil + } + + if len(values) == 0 { + return nil + } + + for _, value := range values[1:] { + if page.Flags().IsSet("Leaf") { + reader := value.Reader() + key_size := ParseUint16(reader, 0) + child_page_id := ParseUint32(reader, 2+int64(key_size)) + fmt.Printf("child_page_id %v\n", child_page_id) + } + } + + return nil +} + +func GetRoot(ctx *ESEContext, value *Value) *ESENT_ROOT_HEADER { + return ctx.Profile.ESENT_ROOT_HEADER(value.Reader(), 0) +} + +func GetBranch(ctx *ESEContext, value *Value) *ESENT_BRANCH_HEADER { + return ctx.Profile.ESENT_BRANCH_HEADER(value.Reader(), 0) +} + +func (self *PageHeader) IsBranch() bool { + return !self.Flags().IsSet("Leaf") +} + +func (self *PageHeader) IsLeaf() bool { + return self.Flags().IsSet("Leaf") +} + +func DumpPage(ctx *ESEContext, id int64) { + header := ctx.GetPage(id) + fmt.Printf("Page %v: %v\n", id, header.DebugString()) + + // Show the tags + values := GetPageValues(ctx, header) + if len(values) == 0 { + return + } + + for _, value := range values { + fmt.Println(value.Tag.DebugString()) + } + + flags := header.Flags() + + if flags.IsSet("Root") { + GetRoot(ctx, values[0]).Dump() + + // Branch header + } else if header.IsBranch() { + GetBranch(ctx, values[0]).Dump() + + // SpaceTree header + } else if flags.IsSet("SpaceTree") { + ctx.Profile.ESENT_SPACE_TREE_HEADER( + &BufferReaderAt{values[0].Buffer}, 0).Dump() + + // Leaf header + } else if header.IsLeaf() { + ctx.Profile.ESENT_LEAF_HEADER( + &BufferReaderAt{values[0].Buffer}, 0).Dump() + } + + for _, value := range values[1:] { + if header.IsBranch() { + ctx.Profile.ESENT_BRANCH_ENTRY(value.Reader(), 0).Dump() + } else if header.IsLeaf() { + if flags.IsSet("SpaceTree") { + ctx.Profile.ESENT_SPACE_TREE_ENTRY(value.Reader(), 0).Dump() + } else if flags.IsSet("Index") { + ctx.Profile.ESENT_INDEX_ENTRY(value.Reader(), 0).Dump() + } else if flags.IsSet("Long") { + // TODO + } else { + ctx.Profile.ESENT_LEAF_ENTRY(value.Reader(), 0).Dump() + } + } + } +} + +func (self *ESENT_ROOT_HEADER) Dump() { + fmt.Println(self.DebugString()) +} + +func (self *ESENT_SPACE_TREE_ENTRY) Dump() { + fmt.Println(self.DebugString()) +} +func (self *ESENT_INDEX_ENTRY) Dump() { + fmt.Println(self.DebugString()) +} + +func (self *ESENT_LEAF_ENTRY) Dump() { + fmt.Println(self.DebugString()) +} + +func (self *ESENT_LEAF_ENTRY) EntryData(value *Value) int64 { + // Tag includes Local Page Key - skip it and the common page key + if value.Tag.Flags()&TAG_COMMON > 0 { + return self.Offset + 4 + + int64(self.LocalPageKeySize()) + } + + return self.Offset + int64(self.CommonPageKeySize()) +} + +func (self *ESENT_BRANCH_HEADER) Dump() { + fmt.Println(self.DebugString()) +} + +func (self *ESENT_BRANCH_ENTRY) Dump() { + fmt.Println(self.DebugString()) + + fmt.Printf(" ChildPageNumber: %#x\n", self.ChildPageNumber()) +} + +func (self *ESENT_SPACE_TREE_HEADER) Dump() { + fmt.Println(self.DebugString()) +} + +func (self *ESENT_LEAF_HEADER) Dump() { + fmt.Println(self.DebugString()) +} + +func (self *ESENT_BRANCH_ENTRY) ChildPageNumber() int64 { + return int64(ParseUint32(self.Reader, self.Offset+int64(self.LocalPageKeySize())+2)) +} + +func WalkPages(ctx *ESEContext, + id int64, + cb func(header *PageHeader, id int64, value *Value)) { + header := ctx.GetPage(id) + values := GetPageValues(ctx, header) + for _, value := range values[1:] { + if header.IsLeaf() { + cb(header, id, value) + } else if header.IsBranch() { + // Walk the branch + branch := ctx.Profile.ESENT_BRANCH_ENTRY(value.Reader(), 0) + WalkPages(ctx, branch.ChildPageNumber(), cb) + } + } +} diff --git a/reader.go b/reader.go new file mode 100644 index 0000000..080c8aa --- /dev/null +++ b/reader.go @@ -0,0 +1,20 @@ +package parser + +type BufferReaderAt struct { + buffer []byte +} + +func (self *BufferReaderAt) ReadAt(buf []byte, offset int64) (int, error) { + to_read := int64(len(buf)) + if offset+to_read > int64(len(self.buffer)) { + to_read = int64(len(self.buffer)) - offset + } + + if to_read < 0 { + return 0, nil + } + + n := copy(buf, self.buffer[offset:offset+to_read]) + + return n, nil +} From eacfe8de87d292587060fe84d5d3eef015e7bf29 Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Fri, 29 Nov 2019 00:42:02 +1000 Subject: [PATCH 07/40] More development --- catalog.go | 220 ++++++++++++++++++++++++++++++++++++++++--- conversion.spec.yaml | 1 + ese_gen.go | 82 ++++++++++++---- ese_profile.json | 15 ++- pages.go | 150 ++++++++++------------------- 5 files changed, 335 insertions(+), 133 deletions(-) diff --git a/catalog.go b/catalog.go index 53dd871..04e25c8 100644 --- a/catalog.go +++ b/catalog.go @@ -1,11 +1,14 @@ +// Parser based on https://github.com/SecureAuthCorp/impacket.git + package parser import ( "errors" "fmt" + "math" + "time" "github.com/Velocidex/ordereddict" - "github.com/davecgh/go-spew/spew" ) const ( @@ -46,9 +49,9 @@ func parseItemName(dd_header *ESENT_DATA_DEFINITION_HEADER) string { } func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) { - leaf_entry := self.ctx.Profile.ESENT_LEAF_ENTRY(value.Reader(), 0) + leaf_entry := NewESENT_LEAF_ENTRY(self.ctx, value) dd_header := self.ctx.Profile.ESENT_DATA_DEFINITION_HEADER( - leaf_entry.Reader, leaf_entry.EntryData(value)) + leaf_entry.Reader, leaf_entry.EntryData()) itemName := parseItemName(dd_header) catalog := dd_header.Catalog() @@ -58,7 +61,7 @@ func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) { table := &Table{ Header: catalog.Table(), Name: itemName, - FatherDataPageNumber: catalog.FDPId(), + FatherDataPageNumber: catalog.Table().FatherDataPageNumber(), Columns: ordereddict.NewDict(), Indexes: ordereddict.NewDict(), LongValues: ordereddict.NewDict()} @@ -66,10 +69,10 @@ func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) { self.Tables.Set(itemName, table) case "CATALOG_TYPE_COLUMN": - self.currentTable.Columns.Set(itemName, catalog.Column()) + self.currentTable.Columns.Set(itemName, catalog) case "CATALOG_TYPE_INDEX": - self.currentTable.Indexes.Set(itemName, catalog.Index()) + self.currentTable.Indexes.Set(itemName, catalog) case "CATALOG_TYPE_LONG_VALUE": } @@ -88,9 +91,10 @@ func (self *Catalog) DumpTable(name string) error { } func (self *Catalog) _walkTableContents(header *PageHeader, id int64, value *Value) { - leaf_entry := self.ctx.Profile.ESENT_LEAF_ENTRY(value.Reader(), 0) - fmt.Printf("Leaf %v\n", leaf_entry.DebugString()) - spew.Dump(value.Buffer) + leaf_entry := NewESENT_LEAF_ENTRY(self.ctx, value) + fmt.Printf("Leaf @ %v \n", id) + _ = leaf_entry + // spew.Dump(value.Buffer) } func (self *Catalog) Dump() { @@ -101,10 +105,10 @@ func (self *Catalog) Dump() { space := " " fmt.Printf("[%v]:\n%sColumns\n", table.Name, space) for idx, column := range table.Columns.Keys() { - column_header_any, _ := table.Columns.Get(column) - column_header := column_header_any.(*CATALOG_TYPE_COLUMN) + catalog_any, _ := table.Columns.Get(column) + catalog := catalog_any.(*ESENT_CATALOG_DATA_DEFINITION_ENTRY) fmt.Printf("%s%s%-5d%-30v%v\n", space, space, idx, - column, column_header.ColumnType().Name) + column, catalog.Column().ColumnType().Name) } fmt.Printf("%sIndexes\n", space) @@ -121,3 +125,195 @@ func ReadCatalog(ctx *ESEContext) *Catalog { WalkPages(ctx, CATALOG_PAGE_NUMBER, result.__addItem) return result } + +type Cursor struct { + ctx *ESEContext + FatherDataPageNumber uint64 + Table *Table + CurrentPageData *PageHeader + CurrentTag int + CurrentValues []*Value +} + +func (self *Catalog) OpenTable(ctx *ESEContext, name string) (*Cursor, error) { + table_any, pres := self.Tables.Get(name) + if !pres { + return nil, errors.New("Table not found") + } + + table := table_any.(*Table) + catalog_entry := table.Header + pageNum := int64(catalog_entry.FatherDataPageNumber()) + done := false + var page *PageHeader + var values []*Value + + for !done { + page = ctx.GetPage(pageNum) + values = GetPageValues(ctx, page) + + if len(values) > 1 { + for _, value := range values[1:] { + if page.IsBranch() { + branchEntry := NewESENT_BRANCH_ENTRY(ctx, value) + pageNum = branchEntry.ChildPageNumber() + break + } else { + done = true + break + } + } + } + } + + return &Cursor{ + ctx: ctx, + Table: table, + FatherDataPageNumber: uint64(catalog_entry.FatherDataPageNumber()), + CurrentPageData: page, + CurrentTag: 0, + CurrentValues: values, + }, nil +} + +func (self *Cursor) GetNextTag() *ESENT_LEAF_ENTRY { + page := self.CurrentPageData + + if self.CurrentTag >= len(self.CurrentValues) { + return nil + } + + if page.IsBranch() { + return nil + } + + page_flags := page.Flags() + if page_flags.IsSet("SpaceTree") || + page_flags.IsSet("Index") || + page_flags.IsSet("Long") { + + // Log this exception. + return nil + } + + return NewESENT_LEAF_ENTRY(self.ctx, self.CurrentValues[self.CurrentTag]) +} + +func (self *Cursor) GetNextRow() *ordereddict.Dict { + self.CurrentTag++ + + tag := self.GetNextTag() + if tag == nil { + page := self.CurrentPageData + if page.NextPageNumber() == 0 { + return nil + } + + self.CurrentPageData = self.ctx.GetPage(int64(page.NextPageNumber())) + self.CurrentTag = 0 + self.CurrentValues = GetPageValues(self.ctx, self.CurrentPageData) + return self.GetNextRow() + } + + return self.tagToRecord(tag) +} + +func (self *Cursor) tagToRecord(tag *ESENT_LEAF_ENTRY) *ordereddict.Dict { + result := ordereddict.NewDict() + + dd_header := self.ctx.Profile.ESENT_DATA_DEFINITION_HEADER(tag.Reader, tag.EntryData()) + fixed_size_offset := dd_header.Offset + self.ctx.Profile. + Off_ESENT_DATA_DEFINITION_HEADER_FixedSizeStart + prevItemLen := int64(0) + variableSizeOffset := dd_header.Offset + int64(dd_header.VariableSizeOffset()) + variableDataBytesProcessed := int64(dd_header.LastVariableDataType()-127) * 2 + + for _, column := range self.Table.Columns.Keys() { + catalog_any, _ := self.Table.Columns.Get(column) + catalog := catalog_any.(*ESENT_CATALOG_DATA_DEFINITION_ENTRY) + + if catalog.Identifier() <= uint32(dd_header.LastFixedSize()) { + space_usage := catalog.Column().SpaceUsage() + + switch catalog.Column().ColumnType().Name { + case "Signed byte": + if space_usage == 1 { + result.Set(column, ParseUint8(tag.Reader, fixed_size_offset)) + } + case "Signed short": + if space_usage == 2 { + result.Set(column, ParseUint16(tag.Reader, fixed_size_offset)) + } + + case "Signed long": + if space_usage == 4 { + result.Set(column, ParseInt32(tag.Reader, fixed_size_offset)) + } + + case "Unsigned long": + if space_usage == 4 { + result.Set(column, ParseUint32(tag.Reader, fixed_size_offset)) + } + + case "Single precision FP": + if space_usage == 4 { + result.Set(column, math.Float32frombits( + ParseUint32(tag.Reader, fixed_size_offset))) + } + + case "Double precision FP": + if space_usage == 8 { + result.Set(column, math.Float64frombits( + ParseUint64(tag.Reader, fixed_size_offset))) + } + + case "DateTime": + if space_usage == 8 { + // Some hair brained time serialization method + // https://docs.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-coltyp + days_since_1900 := math.Float64frombits( + ParseUint64(tag.Reader, fixed_size_offset)) + // In python time.mktime((1900,1,1,0,0,0,0,365,0)) + result.Set(column, time.Unix(int64(days_since_1900*24*60*60)+ + -2208988800, 0)) + } + + case "Long long", "Currency": + if space_usage == 8 { + result.Set(column, ParseUint64(tag.Reader, fixed_size_offset)) + } + + default: + fmt.Printf("Can not handle %v\n", catalog.Column().DebugString()) + } + fixed_size_offset += int64(catalog.Column().SpaceUsage()) + + } else if 127 < catalog.Identifier() && + catalog.Identifier() <= uint32(dd_header.LastVariableDataType()) { + + // Variable data type + index := int64(catalog.Identifier()) - 127 - 1 + itemLen := int64(ParseUint16(tag.Reader, variableSizeOffset+index*2)) + + if itemLen&0x8000 > 0 { + // Empty Item + itemLen = prevItemLen + result.Set(column, nil) + } else { + data := ParseString(tag.Reader, + variableSizeOffset+variableDataBytesProcessed, + itemLen-prevItemLen) + + switch catalog.Column().ColumnType().Name { + case "Text", "Binary": + result.Set(column, data) + } + } + + variableDataBytesProcessed += itemLen - prevItemLen + prevItemLen = itemLen + } + } + + return result +} diff --git a/conversion.spec.yaml b/conversion.spec.yaml index 441f3c7..6f160f4 100644 --- a/conversion.spec.yaml +++ b/conversion.spec.yaml @@ -24,3 +24,4 @@ Structs: - CATALOG_TYPE_LONG_VALUE - ESENT_DATA_DEFINITION_HEADER - ESENT_CATALOG_DATA_DEFINITION_ENTRY + - Misc diff --git a/ese_gen.go b/ese_gen.go index 3f489c3..3b78c2a 100644 --- a/ese_gen.go +++ b/ese_gen.go @@ -68,6 +68,7 @@ type ESEProfile struct { Off_ESENT_DATA_DEFINITION_HEADER_LastVariableDataType int64 Off_ESENT_DATA_DEFINITION_HEADER_VariableSizeOffset int64 Off_ESENT_DATA_DEFINITION_HEADER_Catalog int64 + Off_ESENT_DATA_DEFINITION_HEADER_FixedSizeStart int64 Off_ESENT_INDEX_ENTRY_RecordPageKey int64 Off_ESENT_LEAF_ENTRY_CommonPageKeySize int64 Off_ESENT_LEAF_ENTRY_LocalPageKeySize int64 @@ -94,9 +95,10 @@ type ESEProfile struct { Off_JET_LOGTIME_Year int64 Off_JET_SIGNATURE_Creation int64 Off_JET_SIGNATURE_CreatorMachine int64 + Off_Misc_Misc int64 Off_PageHeader_LastModified int64 - Off_PageHeader_PreviousPage int64 - Off_PageHeader_NextPage int64 + Off_PageHeader_PreviousPageNumber int64 + Off_PageHeader_NextPageNumber int64 Off_PageHeader_FatherPage int64 Off_PageHeader_AvailableDataSize int64 Off_PageHeader_AvailableDataOffset int64 @@ -109,7 +111,7 @@ type ESEProfile struct { func NewESEProfile() *ESEProfile { // Specific offsets can be tweaked to cater for slight version mismatches. - self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,4,0,0,2,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,1,2,3,4,5,4,12,8,16,20,24,28,32,34,36,0,2,2} + self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,4,4,0,-2,0,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,1,2,3,4,5,4,12,0,8,16,20,24,28,32,34,36,0,2,2} return self } @@ -185,6 +187,10 @@ func (self *ESEProfile) JET_SIGNATURE(reader io.ReaderAt, offset int64) *JET_SIG return &JET_SIGNATURE{Reader: reader, Offset: offset, Profile: self} } +func (self *ESEProfile) Misc(reader io.ReaderAt, offset int64) *Misc { + return &Misc{Reader: reader, Offset: offset, Profile: self} +} + func (self *ESEProfile) PageHeader(reader io.ReaderAt, offset int64) *PageHeader { return &PageHeader{Reader: reader, Offset: offset, Profile: self} } @@ -541,12 +547,17 @@ func (self *ESENT_DATA_DEFINITION_HEADER) VariableSizeOffset() uint16 { func (self *ESENT_DATA_DEFINITION_HEADER) Catalog() *ESENT_CATALOG_DATA_DEFINITION_ENTRY { return self.Profile.ESENT_CATALOG_DATA_DEFINITION_ENTRY(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_Catalog + self.Offset) } + +func (self *ESENT_DATA_DEFINITION_HEADER) FixedSizeStart() uint64 { + return ParseUint64(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_FixedSizeStart + self.Offset) +} func (self *ESENT_DATA_DEFINITION_HEADER) DebugString() string { result := fmt.Sprintf("struct ESENT_DATA_DEFINITION_HEADER @ %#x:\n", self.Offset) result += fmt.Sprintf(" LastFixedSize: %#0x\n", self.LastFixedSize()) result += fmt.Sprintf(" LastVariableDataType: %#0x\n", self.LastVariableDataType()) result += fmt.Sprintf(" VariableSizeOffset: %#0x\n", self.VariableSizeOffset()) result += fmt.Sprintf(" Catalog: {\n%v}\n", indent(self.Catalog().DebugString())) + result += fmt.Sprintf(" FixedSizeStart: %#0x\n", self.FixedSizeStart()) return result } @@ -831,6 +842,25 @@ func (self *JET_SIGNATURE) DebugString() string { return result } +type Misc struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *Misc) Size() int { + return 0 +} + +func (self *Misc) Misc() int32 { + return ParseInt32(self.Reader, self.Profile.Off_Misc_Misc + self.Offset) +} +func (self *Misc) DebugString() string { + result := fmt.Sprintf("struct Misc @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Misc: %#0x\n", self.Misc()) + return result +} + type PageHeader struct { Reader io.ReaderAt Offset int64 @@ -845,12 +875,12 @@ func (self *PageHeader) LastModified() *DBTime { return self.Profile.DBTime(self.Reader, self.Profile.Off_PageHeader_LastModified + self.Offset) } -func (self *PageHeader) PreviousPage() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_PageHeader_PreviousPage + self.Offset) +func (self *PageHeader) PreviousPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_PageHeader_PreviousPageNumber + self.Offset) } -func (self *PageHeader) NextPage() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_PageHeader_NextPage + self.Offset) +func (self *PageHeader) NextPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_PageHeader_NextPageNumber + self.Offset) } func (self *PageHeader) FatherPage() uint32 { @@ -874,6 +904,14 @@ func (self *PageHeader) Flags() *Flags { names := make(map[string]bool) + if value & 32 != 0 { + names["SpaceTree"] = true + } + + if value & 64 != 0 { + names["Index"] = true + } + if value & 128 != 0 { names["Long"] = true } @@ -894,22 +932,14 @@ func (self *PageHeader) Flags() *Flags { names["Empty"] = true } - if value & 32 != 0 { - names["SpaceTree"] = true - } - - if value & 64 != 0 { - names["Index"] = true - } - return &Flags{Value: uint64(value), Names: names} } func (self *PageHeader) DebugString() string { result := fmt.Sprintf("struct PageHeader @ %#x:\n", self.Offset) result += fmt.Sprintf(" LastModified: {\n%v}\n", indent(self.LastModified().DebugString())) - result += fmt.Sprintf(" PreviousPage: %#0x\n", self.PreviousPage()) - result += fmt.Sprintf(" NextPage: %#0x\n", self.NextPage()) + result += fmt.Sprintf(" PreviousPageNumber: %#0x\n", self.PreviousPageNumber()) + result += fmt.Sprintf(" NextPageNumber: %#0x\n", self.NextPageNumber()) result += fmt.Sprintf(" FatherPage: %#0x\n", self.FatherPage()) result += fmt.Sprintf(" AvailableDataSize: %#0x\n", self.AvailableDataSize()) result += fmt.Sprintf(" AvailableDataOffset: %#0x\n", self.AvailableDataOffset()) @@ -982,6 +1012,15 @@ func (self Flags) IsSet(flag string) bool { } +func ParseInt32(reader io.ReaderAt, offset int64) int32 { + data := make([]byte, 4) + _, err := reader.ReadAt(data, offset) + if err != nil { + return 0 + } + return int32(binary.LittleEndian.Uint32(data)) +} + func ParseInt8(reader io.ReaderAt, offset int64) int8 { result := make([]byte, 1) _, err := reader.ReadAt(result, offset) @@ -1009,6 +1048,15 @@ func ParseUint32(reader io.ReaderAt, offset int64) uint32 { return binary.LittleEndian.Uint32(data) } +func ParseUint64(reader io.ReaderAt, offset int64) uint64 { + data := make([]byte, 8) + _, err := reader.ReadAt(data, offset) + if err != nil { + return 0 + } + return binary.LittleEndian.Uint64(data) +} + func ParseUint8(reader io.ReaderAt, offset int64) byte { result := make([]byte, 1) _, err := reader.ReadAt(result, offset) diff --git a/ese_profile.json b/ese_profile.json index 13dd554..9e88be5 100644 --- a/ese_profile.json +++ b/ese_profile.json @@ -33,8 +33,8 @@ }], "PageHeader": [0, { "LastModified": [8, ["DBTime"]], - "PreviousPage": [16, ["unsigned long"]], - "NextPage": [20, ["unsigned long"]], + "PreviousPageNumber": [16, ["unsigned long"]], + "NextPageNumber": [20, ["unsigned long"]], "FatherPage": [24, ["unsigned long"]], "AvailableDataSize": [28, ["unsigned short"]], "AvailableDataOffset": [32, ["unsigned short"]], @@ -104,8 +104,8 @@ }], "ESENT_LEAF_ENTRY": [16, { - "CommonPageKeySize": [0, ["unsigned short"]], - "LocalPageKeySize": [2, ["unsigned short"]] + "CommonPageKeySize": [-2, ["unsigned short"]], + "LocalPageKeySize": [0, ["unsigned short"]] }], "ESENT_BRANCH_ENTRY": [16, { @@ -172,7 +172,8 @@ "LastFixedSize": [0, ["char"]], "LastVariableDataType": [1, ["unsigned char"]], "VariableSizeOffset": [2, ["unsigned short"]], - "Catalog": [4, ["ESENT_CATALOG_DATA_DEFINITION_ENTRY"]] + "Catalog": [4, ["ESENT_CATALOG_DATA_DEFINITION_ENTRY"]], + "FixedSizeStart": [4, ["unsigned long long"]] }], "ESENT_CATALOG_DATA_DEFINITION_ENTRY": [0, { @@ -191,5 +192,9 @@ "Table": [10, ["CATALOG_TYPE_TABLE"]], "Index": [10, ["CATALOG_TYPE_INDEX"]], "LongValue": [10, ["CATALOG_TYPE_LONG_VALUE"]] + }], + + "Misc": [0, { + "Misc": [0, ["long"]] }] } diff --git a/pages.go b/pages.go index 02e8e2b..a001b9f 100644 --- a/pages.go +++ b/pages.go @@ -1,7 +1,6 @@ package parser import ( - "errors" "fmt" "io" ) @@ -21,8 +20,8 @@ func (self *Value) Reader() io.ReaderAt { func GetPageValues(ctx *ESEContext, header *PageHeader) []*Value { result := []*Value{} - // Tags are written from the end of the page + // Tags are written from the end of the page offset := ctx.PageSize + header.Offset - 4 for tag_count := header.AvailablePageTag(); tag_count > 0; tag_count-- { @@ -39,85 +38,6 @@ func GetPageValues(ctx *ESEContext, header *PageHeader) []*Value { return result } -func RootNodeVisitor(ctx *ESEContext, id int64) error { - fmt.Printf("RootNodeVisitor: %v\n", id) - - root_page := ctx.GetPage(id) - if !root_page.Flags().IsSet("Root") { - return errors.New(fmt.Sprintf("ID %v: Not root node", id)) - } - - values := GetPageValues(ctx, root_page) - if len(values) == 0 { - return nil - } - - reader := &BufferReaderAt{values[0].Buffer} - root_header := ctx.Profile.ESENT_ROOT_HEADER(reader, 0) - - if root_header.ExtentSpace().Value > 0 { - space_tree_id := int64(root_header.SpaceTreePageNumber()) - if space_tree_id > 0xff000000 { - return errors.New( - fmt.Sprintf("ID %v: Unsupported extent tree", id)) - } - - if space_tree_id > 0 { - // Left node - err := SpaceNodeVisitor(ctx, space_tree_id) - if err != nil { - return err - } - - // Right node - err = SpaceNodeVisitor(ctx, space_tree_id+1) - if err != nil { - return err - } - } - } - - return nil -} - -func SpaceNodeVisitor(ctx *ESEContext, id int64) error { - fmt.Printf("SpaceNodeVisitor: %v\n", id) - - page := ctx.GetPage(id) - if !page.Flags().IsSet("Root") || - !page.Flags().IsSet("SpaceTree") { - return errors.New(fmt.Sprintf("ID %v: Not SpaceTree node", id)) - } - - if page.NextPage() != 0 { - return errors.New(fmt.Sprintf("ID %v: Next Page not 0", id)) - } - - if page.PreviousPage() != 0 { - return errors.New(fmt.Sprintf("ID %v: Prev Page not 0", id)) - } - - values := GetPageValues(ctx, page) - if len(values) == 0 { - return nil - } - - if len(values) == 0 { - return nil - } - - for _, value := range values[1:] { - if page.Flags().IsSet("Leaf") { - reader := value.Reader() - key_size := ParseUint16(reader, 0) - child_page_id := ParseUint32(reader, 2+int64(key_size)) - fmt.Printf("child_page_id %v\n", child_page_id) - } - } - - return nil -} - func GetRoot(ctx *ESEContext, value *Value) *ESENT_ROOT_HEADER { return ctx.Profile.ESENT_ROOT_HEADER(value.Reader(), 0) } @@ -164,13 +84,12 @@ func DumpPage(ctx *ESEContext, id int64) { // Leaf header } else if header.IsLeaf() { - ctx.Profile.ESENT_LEAF_HEADER( - &BufferReaderAt{values[0].Buffer}, 0).Dump() + NewESENT_LEAF_ENTRY(ctx, values[0]).Dump() } for _, value := range values[1:] { if header.IsBranch() { - ctx.Profile.ESENT_BRANCH_ENTRY(value.Reader(), 0).Dump() + NewESENT_BRANCH_ENTRY(ctx, value).Dump() } else if header.IsLeaf() { if flags.IsSet("SpaceTree") { ctx.Profile.ESENT_SPACE_TREE_ENTRY(value.Reader(), 0).Dump() @@ -179,7 +98,7 @@ func DumpPage(ctx *ESEContext, id int64) { } else if flags.IsSet("Long") { // TODO } else { - ctx.Profile.ESENT_LEAF_ENTRY(value.Reader(), 0).Dump() + NewESENT_LEAF_ENTRY(ctx, value).Dump() } } } @@ -196,30 +115,53 @@ func (self *ESENT_INDEX_ENTRY) Dump() { fmt.Println(self.DebugString()) } +// NewESENT_LEAF_ENTRY creates a new ESENT_LEAF_ENTRY +// object. Depending on the Tag flags, there may be present a +// CommonPageKeySize field before the struct. This constructor then +// positions the struct appropriately. +func NewESENT_LEAF_ENTRY(ctx *ESEContext, value *Value) *ESENT_LEAF_ENTRY { + if value.Tag.Flags()&TAG_COMMON > 0 { + // Skip the common header + return ctx.Profile.ESENT_LEAF_ENTRY(value.Reader(), 2) + } + return ctx.Profile.ESENT_LEAF_ENTRY(value.Reader(), 0) +} + func (self *ESENT_LEAF_ENTRY) Dump() { fmt.Println(self.DebugString()) } -func (self *ESENT_LEAF_ENTRY) EntryData(value *Value) int64 { +func (self *ESENT_LEAF_ENTRY) EntryData() int64 { // Tag includes Local Page Key - skip it and the common page key - if value.Tag.Flags()&TAG_COMMON > 0 { - return self.Offset + 4 + - int64(self.LocalPageKeySize()) - } - - return self.Offset + int64(self.CommonPageKeySize()) + return self.Offset + 2 + int64(self.LocalPageKeySize()) } func (self *ESENT_BRANCH_HEADER) Dump() { fmt.Println(self.DebugString()) } -func (self *ESENT_BRANCH_ENTRY) Dump() { - fmt.Println(self.DebugString()) +// NewESENT_BRANCH_ENTRY creates a new ESENT_BRANCH_ENTRY +// object. Depending on the Tag flags, there may be present a +// CommonPageKeySize field before the struct. This construstor then +// positions the struct appropriately. +func NewESENT_BRANCH_ENTRY(ctx *ESEContext, value *Value) *ESENT_BRANCH_ENTRY { + if value.Tag.Flags()&TAG_COMMON > 0 { + // Skip the common header + return ctx.Profile.ESENT_BRANCH_ENTRY(value.Reader(), 2) + } + return ctx.Profile.ESENT_BRANCH_ENTRY(value.Reader(), 0) +} +func (self *ESENT_BRANCH_ENTRY) Dump() { + fmt.Printf("%s", self.DebugString()) fmt.Printf(" ChildPageNumber: %#x\n", self.ChildPageNumber()) } +func (self *ESENT_BRANCH_ENTRY) ChildPageNumber() int64 { + return int64(ParseUint32(self.Reader, self.Offset+2+ + int64(self.LocalPageKeySize()))) +} + func (self *ESENT_SPACE_TREE_HEADER) Dump() { fmt.Println(self.DebugString()) } @@ -228,22 +170,32 @@ func (self *ESENT_LEAF_HEADER) Dump() { fmt.Println(self.DebugString()) } -func (self *ESENT_BRANCH_ENTRY) ChildPageNumber() int64 { - return int64(ParseUint32(self.Reader, self.Offset+int64(self.LocalPageKeySize())+2)) -} - func WalkPages(ctx *ESEContext, id int64, cb func(header *PageHeader, id int64, value *Value)) { + if id <= 0 { + return + } + header := ctx.GetPage(id) values := GetPageValues(ctx, header) + + // No more records. + if len(values) == 0 { + return + } + for _, value := range values[1:] { if header.IsLeaf() { cb(header, id, value) } else if header.IsBranch() { // Walk the branch - branch := ctx.Profile.ESENT_BRANCH_ENTRY(value.Reader(), 0) + branch := NewESENT_BRANCH_ENTRY(ctx, value) WalkPages(ctx, branch.ChildPageNumber(), cb) } } + + if header.NextPageNumber() > 0 { + WalkPages(ctx, int64(header.NextPageNumber()), cb) + } } From e04cb2fad3049cbe64fcc8068c99b7baffe2969d Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Fri, 29 Nov 2019 10:50:39 +1000 Subject: [PATCH 08/40] Implemented Dump Table. --- catalog.go | 328 +++++++++++++++++++---------------------------- ese_gen.go | 40 +++++- ese_profile.json | 4 +- pages.go | 28 +++- 4 files changed, 191 insertions(+), 209 deletions(-) diff --git a/catalog.go b/catalog.go index 04e25c8..32fa21d 100644 --- a/catalog.go +++ b/catalog.go @@ -16,6 +16,7 @@ const ( ) type Table struct { + ctx *ESEContext Header *CATALOG_TYPE_TABLE FatherDataPageNumber uint32 Name string @@ -24,201 +25,8 @@ type Table struct { LongValues *ordereddict.Dict } -type Catalog struct { - ctx *ESEContext - - Tables *ordereddict.Dict - - currentTable *Table -} - -func parseItemName(dd_header *ESENT_DATA_DEFINITION_HEADER) string { - last_variable_data_type := int64(dd_header.LastVariableDataType()) - numEntries := last_variable_data_type - - if last_variable_data_type > 127 { - numEntries = last_variable_data_type - 127 - } - - itemLen := ParseUint16(dd_header.Reader, - dd_header.Offset+int64(dd_header.VariableSizeOffset())) - - return ParseString(dd_header.Reader, - dd_header.Offset+int64(dd_header.VariableSizeOffset())+ - 2*numEntries, int64(itemLen)) -} - -func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) { - leaf_entry := NewESENT_LEAF_ENTRY(self.ctx, value) - dd_header := self.ctx.Profile.ESENT_DATA_DEFINITION_HEADER( - leaf_entry.Reader, leaf_entry.EntryData()) - - itemName := parseItemName(dd_header) - catalog := dd_header.Catalog() - - switch catalog.Type().Name { - case "CATALOG_TYPE_TABLE": - table := &Table{ - Header: catalog.Table(), - Name: itemName, - FatherDataPageNumber: catalog.Table().FatherDataPageNumber(), - Columns: ordereddict.NewDict(), - Indexes: ordereddict.NewDict(), - LongValues: ordereddict.NewDict()} - self.currentTable = table - self.Tables.Set(itemName, table) - - case "CATALOG_TYPE_COLUMN": - self.currentTable.Columns.Set(itemName, catalog) - - case "CATALOG_TYPE_INDEX": - self.currentTable.Indexes.Set(itemName, catalog) - case "CATALOG_TYPE_LONG_VALUE": - - } -} - -func (self *Catalog) DumpTable(name string) error { - table_any, pres := self.Tables.Get(name) - if !pres { - return errors.New("Table not found") - } - - table := table_any.(*Table) - WalkPages(self.ctx, int64(table.FatherDataPageNumber), self._walkTableContents) - - return nil -} - -func (self *Catalog) _walkTableContents(header *PageHeader, id int64, value *Value) { - leaf_entry := NewESENT_LEAF_ENTRY(self.ctx, value) - fmt.Printf("Leaf @ %v \n", id) - _ = leaf_entry - // spew.Dump(value.Buffer) -} - -func (self *Catalog) Dump() { - for _, name := range self.Tables.Keys() { - table_any, _ := self.Tables.Get(name) - table := table_any.(*Table) - - space := " " - fmt.Printf("[%v]:\n%sColumns\n", table.Name, space) - for idx, column := range table.Columns.Keys() { - catalog_any, _ := table.Columns.Get(column) - catalog := catalog_any.(*ESENT_CATALOG_DATA_DEFINITION_ENTRY) - fmt.Printf("%s%s%-5d%-30v%v\n", space, space, idx, - column, catalog.Column().ColumnType().Name) - } - - fmt.Printf("%sIndexes\n", space) - for _, index := range table.Indexes.Keys() { - fmt.Printf("%s%s%v:\n", space, space, index) - } - fmt.Printf("\n") - } -} - -func ReadCatalog(ctx *ESEContext) *Catalog { - result := &Catalog{ctx: ctx, Tables: ordereddict.NewDict()} - - WalkPages(ctx, CATALOG_PAGE_NUMBER, result.__addItem) - return result -} - -type Cursor struct { - ctx *ESEContext - FatherDataPageNumber uint64 - Table *Table - CurrentPageData *PageHeader - CurrentTag int - CurrentValues []*Value -} - -func (self *Catalog) OpenTable(ctx *ESEContext, name string) (*Cursor, error) { - table_any, pres := self.Tables.Get(name) - if !pres { - return nil, errors.New("Table not found") - } - - table := table_any.(*Table) - catalog_entry := table.Header - pageNum := int64(catalog_entry.FatherDataPageNumber()) - done := false - var page *PageHeader - var values []*Value - - for !done { - page = ctx.GetPage(pageNum) - values = GetPageValues(ctx, page) - - if len(values) > 1 { - for _, value := range values[1:] { - if page.IsBranch() { - branchEntry := NewESENT_BRANCH_ENTRY(ctx, value) - pageNum = branchEntry.ChildPageNumber() - break - } else { - done = true - break - } - } - } - } - - return &Cursor{ - ctx: ctx, - Table: table, - FatherDataPageNumber: uint64(catalog_entry.FatherDataPageNumber()), - CurrentPageData: page, - CurrentTag: 0, - CurrentValues: values, - }, nil -} - -func (self *Cursor) GetNextTag() *ESENT_LEAF_ENTRY { - page := self.CurrentPageData - - if self.CurrentTag >= len(self.CurrentValues) { - return nil - } - - if page.IsBranch() { - return nil - } - - page_flags := page.Flags() - if page_flags.IsSet("SpaceTree") || - page_flags.IsSet("Index") || - page_flags.IsSet("Long") { - - // Log this exception. - return nil - } - - return NewESENT_LEAF_ENTRY(self.ctx, self.CurrentValues[self.CurrentTag]) -} - -func (self *Cursor) GetNextRow() *ordereddict.Dict { - self.CurrentTag++ - - tag := self.GetNextTag() - if tag == nil { - page := self.CurrentPageData - if page.NextPageNumber() == 0 { - return nil - } - - self.CurrentPageData = self.ctx.GetPage(int64(page.NextPageNumber())) - self.CurrentTag = 0 - self.CurrentValues = GetPageValues(self.ctx, self.CurrentPageData) - return self.GetNextRow() - } - - return self.tagToRecord(tag) -} - -func (self *Cursor) tagToRecord(tag *ESENT_LEAF_ENTRY) *ordereddict.Dict { +// Code based on +func (self *Table) tagToRecord(tag *ESENT_LEAF_ENTRY) *ordereddict.Dict { result := ordereddict.NewDict() dd_header := self.ctx.Profile.ESENT_DATA_DEFINITION_HEADER(tag.Reader, tag.EntryData()) @@ -228,19 +36,30 @@ func (self *Cursor) tagToRecord(tag *ESENT_LEAF_ENTRY) *ordereddict.Dict { variableSizeOffset := dd_header.Offset + int64(dd_header.VariableSizeOffset()) variableDataBytesProcessed := int64(dd_header.LastVariableDataType()-127) * 2 - for _, column := range self.Table.Columns.Keys() { - catalog_any, _ := self.Table.Columns.Get(column) + for _, column := range self.Columns.Keys() { + catalog_any, _ := self.Columns.Get(column) catalog := catalog_any.(*ESENT_CATALOG_DATA_DEFINITION_ENTRY) if catalog.Identifier() <= uint32(dd_header.LastFixedSize()) { space_usage := catalog.Column().SpaceUsage() switch catalog.Column().ColumnType().Name { + case "Boolean": + if space_usage == 1 { + result.Set(column, ParseUint8(tag.Reader, fixed_size_offset) > 0) + } + case "Signed byte": if space_usage == 1 { result.Set(column, ParseUint8(tag.Reader, fixed_size_offset)) } + case "Signed short": + if space_usage == 2 { + result.Set(column, ParseInt16(tag.Reader, fixed_size_offset)) + } + + case "Unsigned short": if space_usage == 2 { result.Set(column, ParseUint16(tag.Reader, fixed_size_offset)) } @@ -317,3 +136,118 @@ func (self *Cursor) tagToRecord(tag *ESENT_LEAF_ENTRY) *ordereddict.Dict { return result } + +// DumpTable extracts all rows in the named table and passes them into +// the callback. The callback may cancel the walk at any time by +// returning an error which is passed to our caller. +func (self *Catalog) DumpTable(name string, cb func(row *ordereddict.Dict) error) error { + table_any, pres := self.Tables.Get(name) + if !pres { + return errors.New("Table not found") + } + + table := table_any.(*Table) + err := WalkPages(self.ctx, int64(table.FatherDataPageNumber), + func(header *PageHeader, id int64, value *Value) error { + leaf_entry := NewESENT_LEAF_ENTRY(self.ctx, value) + return cb(table.tagToRecord(leaf_entry)) + }) + if err != nil { + return err + } + return nil +} + +// Catalog represents the database's catalog. +type Catalog struct { + ctx *ESEContext + + Tables *ordereddict.Dict + + currentTable *Table +} + +func parseItemName(dd_header *ESENT_DATA_DEFINITION_HEADER) string { + last_variable_data_type := int64(dd_header.LastVariableDataType()) + numEntries := last_variable_data_type + + if last_variable_data_type > 127 { + numEntries = last_variable_data_type - 127 + } + + itemLen := ParseUint16(dd_header.Reader, + dd_header.Offset+int64(dd_header.VariableSizeOffset())) + + return ParseString(dd_header.Reader, + dd_header.Offset+int64(dd_header.VariableSizeOffset())+ + 2*numEntries, int64(itemLen)) +} + +func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) error { + leaf_entry := NewESENT_LEAF_ENTRY(self.ctx, value) + dd_header := self.ctx.Profile.ESENT_DATA_DEFINITION_HEADER( + leaf_entry.Reader, leaf_entry.EntryData()) + + itemName := parseItemName(dd_header) + catalog := dd_header.Catalog() + + switch catalog.Type().Name { + case "CATALOG_TYPE_TABLE": + table := &Table{ + ctx: self.ctx, + Header: catalog.Table(), + Name: itemName, + FatherDataPageNumber: catalog.Table().FatherDataPageNumber(), + Columns: ordereddict.NewDict(), + Indexes: ordereddict.NewDict(), + LongValues: ordereddict.NewDict()} + self.currentTable = table + self.Tables.Set(itemName, table) + + case "CATALOG_TYPE_COLUMN": + self.currentTable.Columns.Set(itemName, catalog) + + case "CATALOG_TYPE_INDEX": + self.currentTable.Indexes.Set(itemName, catalog) + case "CATALOG_TYPE_LONG_VALUE": + + } + + return nil +} + +func (self *Catalog) Dump() string { + result := "" + + for _, name := range self.Tables.Keys() { + table_any, _ := self.Tables.Get(name) + table := table_any.(*Table) + + space := " " + result += fmt.Sprintf("[%v]:\n%sColumns\n", table.Name, space) + for idx, column := range table.Columns.Keys() { + catalog_any, _ := table.Columns.Get(column) + catalog := catalog_any.(*ESENT_CATALOG_DATA_DEFINITION_ENTRY) + result += fmt.Sprintf("%s%s%-5d%-30v%v\n", space, space, idx, + column, catalog.Column().ColumnType().Name) + } + + result += fmt.Sprintf("%sIndexes\n", space) + for _, index := range table.Indexes.Keys() { + result += fmt.Sprintf("%s%s%v:\n", space, space, index) + } + result += "\n" + } + + return result +} + +func ReadCatalog(ctx *ESEContext) (*Catalog, error) { + result := &Catalog{ctx: ctx, Tables: ordereddict.NewDict()} + + err := WalkPages(ctx, CATALOG_PAGE_NUMBER, result.__addItem) + if err != nil { + return nil, err + } + return result, nil +} diff --git a/ese_gen.go b/ese_gen.go index 3b78c2a..0661e30 100644 --- a/ese_gen.go +++ b/ese_gen.go @@ -96,6 +96,8 @@ type ESEProfile struct { Off_JET_SIGNATURE_Creation int64 Off_JET_SIGNATURE_CreatorMachine int64 Off_Misc_Misc int64 + Off_Misc_Misc2 int64 + Off_Misc_Misc3 int64 Off_PageHeader_LastModified int64 Off_PageHeader_PreviousPageNumber int64 Off_PageHeader_NextPageNumber int64 @@ -111,7 +113,7 @@ type ESEProfile struct { func NewESEProfile() *ESEProfile { // Specific offsets can be tweaked to cater for slight version mismatches. - self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,4,4,0,-2,0,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,1,2,3,4,5,4,12,0,8,16,20,24,28,32,34,36,0,2,2} + self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,4,4,0,-2,0,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,1,2,3,4,5,4,12,0,0,0,8,16,20,24,28,32,34,36,0,2,2} return self } @@ -855,9 +857,19 @@ func (self *Misc) Size() int { func (self *Misc) Misc() int32 { return ParseInt32(self.Reader, self.Profile.Off_Misc_Misc + self.Offset) } + +func (self *Misc) Misc2() int16 { + return ParseInt16(self.Reader, self.Profile.Off_Misc_Misc2 + self.Offset) +} + +func (self *Misc) Misc3() int64 { + return int64(ParseUint64(self.Reader, self.Profile.Off_Misc_Misc3 + self.Offset)) +} func (self *Misc) DebugString() string { result := fmt.Sprintf("struct Misc @ %#x:\n", self.Offset) result += fmt.Sprintf(" Misc: %#0x\n", self.Misc()) + result += fmt.Sprintf(" Misc2: %#0x\n", self.Misc2()) + result += fmt.Sprintf(" Misc3: %#0x\n", self.Misc3()) return result } @@ -904,6 +916,10 @@ func (self *PageHeader) Flags() *Flags { names := make(map[string]bool) + if value & 8 != 0 { + names["Empty"] = true + } + if value & 32 != 0 { names["SpaceTree"] = true } @@ -928,10 +944,6 @@ func (self *PageHeader) Flags() *Flags { names["Parent"] = true } - if value & 8 != 0 { - names["Empty"] = true - } - return &Flags{Value: uint64(value), Names: names} } @@ -1012,6 +1024,15 @@ func (self Flags) IsSet(flag string) bool { } +func ParseInt16(reader io.ReaderAt, offset int64) int16 { + data := make([]byte, 2) + _, err := reader.ReadAt(data, offset) + if err != nil { + return 0 + } + return int16(binary.LittleEndian.Uint16(data)) +} + func ParseInt32(reader io.ReaderAt, offset int64) int32 { data := make([]byte, 4) _, err := reader.ReadAt(data, offset) @@ -1021,6 +1042,15 @@ func ParseInt32(reader io.ReaderAt, offset int64) int32 { return int32(binary.LittleEndian.Uint32(data)) } +func ParseInt64(reader io.ReaderAt, offset int64) int64 { + data := make([]byte, 8) + _, err := reader.ReadAt(data, offset) + if err != nil { + return 0 + } + return int64(binary.LittleEndian.Uint64(data)) +} + func ParseInt8(reader io.ReaderAt, offset int64) int8 { result := make([]byte, 1) _, err := reader.ReadAt(result, offset) diff --git a/ese_profile.json b/ese_profile.json index 9e88be5..a616439 100644 --- a/ese_profile.json +++ b/ese_profile.json @@ -195,6 +195,8 @@ }], "Misc": [0, { - "Misc": [0, ["long"]] + "Misc": [0, ["long"]], + "Misc2": [0, ["short"]], + "Misc3": [0, ["long long"]] }] } diff --git a/pages.go b/pages.go index a001b9f..055e6f5 100644 --- a/pages.go +++ b/pages.go @@ -170,11 +170,15 @@ func (self *ESENT_LEAF_HEADER) Dump() { fmt.Println(self.DebugString()) } +// WalkPages walks the b tree starting with the page id specified and +// extracts all tagged values into the callback. The callback may +// return an error which will cause WalkPages to stop and relay that +// error to our caller. func WalkPages(ctx *ESEContext, id int64, - cb func(header *PageHeader, id int64, value *Value)) { + cb func(header *PageHeader, page_id int64, value *Value) error) error { if id <= 0 { - return + return nil } header := ctx.GetPage(id) @@ -182,20 +186,32 @@ func WalkPages(ctx *ESEContext, // No more records. if len(values) == 0 { - return + return nil } for _, value := range values[1:] { if header.IsLeaf() { - cb(header, id, value) + // Allow the callback to return early (e.g. in case of cancellation) + err := cb(header, id, value) + if err != nil { + return err + } } else if header.IsBranch() { // Walk the branch branch := NewESENT_BRANCH_ENTRY(ctx, value) - WalkPages(ctx, branch.ChildPageNumber(), cb) + err := WalkPages(ctx, branch.ChildPageNumber(), cb) + if err != nil { + return err + } } } if header.NextPageNumber() > 0 { - WalkPages(ctx, int64(header.NextPageNumber()), cb) + err := WalkPages(ctx, int64(header.NextPageNumber()), cb) + if err != nil { + return err + } } + + return nil } From 4494ac6c29c6beb2ddd31591fd10617d9a7329e9 Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Mon, 2 Dec 2019 16:41:26 +1000 Subject: [PATCH 09/40] Support WebCache files. --- README.md | 172 +++++++++++++++++++++++++++++ catalog.go | 256 +++++++++++++++++++++++++++++++++++++------ context.go | 22 +++- conversion.spec.yaml | 3 + debug.go | 9 ++ ese_gen.go | 142 ++++++++++++++++++------ ese_profile.json | 38 +++---- pages.go | 69 ++++++++++-- reader.go | 5 + 9 files changed, 615 insertions(+), 101 deletions(-) create mode 100644 README.md create mode 100644 debug.go diff --git a/README.md b/README.md new file mode 100644 index 0000000..c89ffb2 --- /dev/null +++ b/README.md @@ -0,0 +1,172 @@ +# A Go based ESE parser. + +The Extensible Storage Engine (ESE) Database File is commonly used +within Windows to store various application specific information. It +is the Microsoft analogue to sqlite - so just like sqlite is used to +store chrome history, ESE is used to store Internet Explorer history. + +In essence it is a flat file database. This project is a library to +help read such a file. The following description is a high lieve +account of the main feautures of the file format and how to access +these using the library. + +## File format overview + +The file consists of pages. The page size can vary but it is specified +in the file header. + +The file may contain multiple objects (tables) stored within +pages. The pages form a B tree where data is stored in the actual leaf +pages. As the file grows the tree canbe extended by inserting pages +into it. + +Data is stored inside each page in a `Tag`. Tags are just a series of +data blobs stored in each page. + +The WalkPages() function can be used to produce all the tags starting +at a root page. The function walks the B+ tree automatically and +parses out each tag. The callback is called for each tag - if the +callback returns an error the walk is stopped and the error is relayed +to the WalkPages() caller. + +## The page + +Each page contains a series of tags. You can see help about a specific +page using the `page` command: + +``` +$ eseparser page WebCacheV01.dat 4 +Page 4: struct PageHeader @ 0x28000: + LastModified: { + struct DBTime @ 0x28008: + Hours: 0xfbf4 + Min: 0x0 + Sec: 0x0 + } + PreviousPageNumber: 0x0 + NextPageNumber: 0x0 + FatherPage: 0x2 + AvailableDataSize: 0x7f3b + AvailableDataOffset: 0x5d + AvailablePageTag: 0x6 + Flags: 108549 (Parent,Root) + +Tag 0 @ 0x2fffc offset 0x0 length 0x10 +Tag 1 @ 0x2fff8 offset 0x16 length 0x13 +Tag 2 @ 0x2fff4 offset 0x29 length 0xe +Tag 3 @ 0x2fff0 offset 0x37 length 0x13 +Tag 4 @ 0x2ffec offset 0x4a length 0x13 +Tag 5 @ 0x2ffe8 offset 0x10 length 0x6 +struct ESENT_ROOT_HEADER @ 0x0: + InitialNumberOfPages: 0x14 + ParentFDP: 0x1 + ExtentSpace: Multiple (1) + SpaceTreePageNumber: 0x5 + +struct ESENT_BRANCH_ENTRY @ 0x0: + LocalPageKeySize: 0xd + ChildPageNumber: 0xd +struct ESENT_BRANCH_ENTRY @ 0x0: + LocalPageKeySize: 0x8 + ChildPageNumber: 0xe +struct ESENT_BRANCH_ENTRY @ 0x0: + LocalPageKeySize: 0xd + ChildPageNumber: 0x13 +struct ESENT_BRANCH_ENTRY @ 0x0: + LocalPageKeySize: 0xd + ChildPageNumber: 0x14 +struct ESENT_BRANCH_ENTRY @ 0x0: + LocalPageKeySize: 0x0 + ChildPageNumber: 0x16 +``` + +The example above shows a root page (4) containing 5 branch nodes. + +## The catalog + +The ESE file contains a catalog starting from page 4. The catalog +defines all the tables, their columns and types stat are stored in the +database. + +You can see the catalong by runing the `catalog` command: + +``` +$ eseparser catalog /shared/WebCacheV01.dat +[MSysObjects] (FDP 0x4): + Columns + 0 ObjidTable Signed long + 1 Type Signed short + 2 Id Signed long + 3 ColtypOrPgnoFDP Signed long + 4 SpaceUsage Signed long + 5 Flags Signed long + 6 PagesOrLocale Signed long + 7 RootFlag Boolean + 8 RecordOffset Signed short +``` + +The first table in the catalog called `MSysObjects` is really a +database table containing a description of all the tables in the file. + +## Tables + +Ultimately the ESE format is a database storage engine and it stores +rows in tables. Each table is stored inside the B tree rooted by the +DFP ID shown in the catalog. Each row is stored inside a tag (inside +one of the pages within the tree). + +There are three types of columns: + +- Fixed size (e.g. integers) have a known size. +- Variable size (e.g. Strings) have a variable size. +- Tagged data - these columns are often null and therefore may not be + present. The database stored these with their column ID as a map. + +Therefore within the tag for each column, there are three distinct +storage areas. You can see how each record is parsed using the --debug flag: + +``` +$ eseparser dump WebCacheV01.dat MSysObjects --debug +Walking page 4 +Got 6 values for page 4 +Walking page 13 +Got 404 values for page 13 +Processing row in Tag @ 491512 0xd (0x37)([]uint8) (len=55 cap=55) { + 00000000 07 00 06 00 01 7f 80 00 00 02 08 80 20 00 02 00 |............ ...| + 00000010 00 00 01 00 02 00 00 00 04 00 00 00 50 00 00 00 |............P...| + 00000020 00 00 00 c0 14 00 00 00 ff 00 0b 00 4d 53 79 73 |............MSys| + 00000030 4f 62 6a 65 63 74 73 |Objects| +} +struct ESENT_LEAF_ENTRY @ 0x2: + CommonPageKeySize: 0x7 + LocalPageKeySize: 0x6 + +struct ESENT_DATA_DEFINITION_HEADER @ 0xa: + LastFixedType: 0x8 + LastVariableDataType: 0x80 + VariableSizeOffset: 0x20 + +Column ObjidTable Identifier 1 Type Signed long +Consume 0x4 bytes of FIXED space from 0xe +Column Type Identifier 2 Type Signed short +Consume 0x2 bytes of FIXED space from 0x12 +Column Id Identifier 3 Type Signed long +... +{"ObjidTable":2,"Type":1,"Id":2,"ColtypOrPgnoFDP":4,"SpaceUsage":80,"Flags":-1073741824,"PagesOrLocale":20,"RootFlag":true,"Name":"MSysObjects"} +``` + +If you just want to dump out the columns omit the `--debug` flag: +``` +$ ./eseparser dump /shared/WebCacheV01.dat HstsEntryEx_46 +{"EntryId":1,"MinimizedRDomainHash":0,"MinimizedRDomainLength":8,"IncludeSubdomains":177,"Expires":9223372036854775807,"LastTimeUsed":9223372036854775807,"RDomain":":version"} +{"EntryId":2,"MinimizedRDomainHash":1536723792475384667,"MinimizedRDomainLength":10,"IncludeSubdomains":1,"Expires":132508317752329580,"LastTimeUsed":132192957752329580,"RDomain":"com.office.www"} +``` + + + +References: + * https://github.com/SecureAuthCorp/impacket.git + * https://blogs.technet.microsoft.com/askds/2017/12/04/ese-deep-dive-part-1-the-anatomy-of-an-ese-database/ + * https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2012-R2-and-2012/hh875546(v=ws.11)?redirectedfrom=MSDN + * http://hh.diva-portal.org/smash/get/diva2:635743/FULLTEXT02.pdf + * https://github.com/libyal/libesedb/blob/master/documentation/Extensible%20Storage%20Engine%20(ESE)%20Database%20File%20(EDB)%20format.asciidoc diff --git a/catalog.go b/catalog.go index 32fa21d..39f9d19 100644 --- a/catalog.go +++ b/catalog.go @@ -9,6 +9,7 @@ import ( "time" "github.com/Velocidex/ordereddict" + "github.com/davecgh/go-spew/spew" ) const ( @@ -25,65 +26,132 @@ type Table struct { LongValues *ordereddict.Dict } -// Code based on -func (self *Table) tagToRecord(tag *ESENT_LEAF_ENTRY) *ordereddict.Dict { +// The tag contains a single row. +// 00000000 09 00 7f 80 00 00 00 00 00 00 01 06 7f 2d 00 01 |.............-..| +// 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................| +// 00000020 00 00 00 b1 00 00 00 ff ff ff ff ff ff ff 7f ff |................| +// 00000030 ff ff ff ff ff ff 7f c0 00 01 04 00 01 3a 00 76 |.............:.v| +// 00000040 00 65 00 72 00 73 00 69 00 6f 00 6e 00 00 00 |.e.r.s.i.o.n...| + +// The key consumes the first 11 bytes: +// struct ESENT_LEAF_ENTRY @ 0x0: +// CommonPageKeySize: 0x0 +// LocalPageKeySize: 0x9 + +// Followed by a data definition header: +//struct ESENT_DATA_DEFINITION_HEADER @ 0xb: +// LastFixedType: 0x6 +// LastVariableDataType: 0x7f +// VariableSizeOffset: 0x2d + +// Column IDs below LastFixedSize will be stored in the fixed size +// portion. Column id below LastVariableDataType will be stored in the +// variable data section and higher than LastVariableDataType will be +// tagged. + +// The fixed section starts immediately after the +// ESENT_DATA_DEFINITION_HEADER (offset 0xb + 4 = 0xf) + +// Then the following columns consume their types: +// Column EntryId Identifier 1 Type Long long +// Column MinimizedRDomainHash Identifier 2 Type Long long +// Column MinimizedRDomainLength Identifier 3 Type Unsigned long +// Column IncludeSubdomains Identifier 4 Type Unsigned long +// Column Expires Identifier 5 Type Long long +// Column LastTimeUsed Identifier 6 Type Long long + +// In the above example we have no variable sized columns, so we go +// straight to the tagged values: + +// Then the tagged values are consumed +// Column RDomain Identifier 256 Type Long Text + +func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { + tag := NewESENT_LEAF_ENTRY(self.ctx, value) + + if Debug { + fmt.Printf("Processing row in Tag @ %d %#x (%#x)", + value.Tag.Offset, value.Tag.ValueOffset(self.ctx), + value.Tag.ValueSize(self.ctx)) + spew.Dump(value.Buffer) + tag.Dump() + } + result := ordereddict.NewDict() + var taggedItems map[uint32][]byte + dd_header := self.ctx.Profile.ESENT_DATA_DEFINITION_HEADER(tag.Reader, tag.EntryData()) - fixed_size_offset := dd_header.Offset + self.ctx.Profile. - Off_ESENT_DATA_DEFINITION_HEADER_FixedSizeStart + + // Start to parse immediately after the dd_header + offset := dd_header.Offset + int64(dd_header.Size()) + + if Debug { + fmt.Println(dd_header.DebugString()) + } + prevItemLen := int64(0) variableSizeOffset := dd_header.Offset + int64(dd_header.VariableSizeOffset()) variableDataBytesProcessed := int64(dd_header.LastVariableDataType()-127) * 2 + // Iterate over the column definitions and decode each + // identifier according to where it comes from. for _, column := range self.Columns.Keys() { catalog_any, _ := self.Columns.Get(column) catalog := catalog_any.(*ESENT_CATALOG_DATA_DEFINITION_ENTRY) + identifier := catalog.Identifier() + column_type := catalog.Column().ColumnType().Name - if catalog.Identifier() <= uint32(dd_header.LastFixedSize()) { + if Debug { + fmt.Printf("Column %v Identifier %v Type %v\n", column, identifier, + catalog.Column().ColumnType().Name) + } + + // Column is stored in the fixed section. + if identifier <= uint32(dd_header.LastFixedType()) { space_usage := catalog.Column().SpaceUsage() - switch catalog.Column().ColumnType().Name { + switch column_type { case "Boolean": if space_usage == 1 { - result.Set(column, ParseUint8(tag.Reader, fixed_size_offset) > 0) + result.Set(column, ParseUint8(tag.Reader, offset) > 0) } case "Signed byte": if space_usage == 1 { - result.Set(column, ParseUint8(tag.Reader, fixed_size_offset)) + result.Set(column, ParseUint8(tag.Reader, offset)) } case "Signed short": if space_usage == 2 { - result.Set(column, ParseInt16(tag.Reader, fixed_size_offset)) + result.Set(column, ParseInt16(tag.Reader, offset)) } case "Unsigned short": if space_usage == 2 { - result.Set(column, ParseUint16(tag.Reader, fixed_size_offset)) + result.Set(column, ParseUint16(tag.Reader, offset)) } case "Signed long": if space_usage == 4 { - result.Set(column, ParseInt32(tag.Reader, fixed_size_offset)) + result.Set(column, ParseInt32(tag.Reader, offset)) } case "Unsigned long": if space_usage == 4 { - result.Set(column, ParseUint32(tag.Reader, fixed_size_offset)) + result.Set(column, ParseUint32(tag.Reader, offset)) } case "Single precision FP": if space_usage == 4 { result.Set(column, math.Float32frombits( - ParseUint32(tag.Reader, fixed_size_offset))) + ParseUint32(tag.Reader, offset))) } case "Double precision FP": if space_usage == 8 { result.Set(column, math.Float64frombits( - ParseUint64(tag.Reader, fixed_size_offset))) + ParseUint64(tag.Reader, offset))) } case "DateTime": @@ -91,7 +159,7 @@ func (self *Table) tagToRecord(tag *ESENT_LEAF_ENTRY) *ordereddict.Dict { // Some hair brained time serialization method // https://docs.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-coltyp days_since_1900 := math.Float64frombits( - ParseUint64(tag.Reader, fixed_size_offset)) + ParseUint64(tag.Reader, offset)) // In python time.mktime((1900,1,1,0,0,0,0,365,0)) result.Set(column, time.Unix(int64(days_since_1900*24*60*60)+ -2208988800, 0)) @@ -99,19 +167,28 @@ func (self *Table) tagToRecord(tag *ESENT_LEAF_ENTRY) *ordereddict.Dict { case "Long long", "Currency": if space_usage == 8 { - result.Set(column, ParseUint64(tag.Reader, fixed_size_offset)) + result.Set(column, ParseUint64(tag.Reader, offset)) } default: - fmt.Printf("Can not handle %v\n", catalog.Column().DebugString()) + fmt.Printf("Can not handle Column %v fixed data %v\n", + column, catalog.Column().DebugString()) + } + + if Debug { + fmt.Printf("Consumed %#x bytes of FIXED space from %#x\n", + catalog.Column().SpaceUsage(), offset) } - fixed_size_offset += int64(catalog.Column().SpaceUsage()) - } else if 127 < catalog.Identifier() && - catalog.Identifier() <= uint32(dd_header.LastVariableDataType()) { + // Move our offset along + offset += int64(catalog.Column().SpaceUsage()) + + // Identifier is stored in the variable section + } else if 127 < identifier && + identifier <= uint32(dd_header.LastVariableDataType()) { // Variable data type - index := int64(catalog.Identifier()) - 127 - 1 + index := int64(identifier) - 127 - 1 itemLen := int64(ParseUint16(tag.Reader, variableSizeOffset+index*2)) if itemLen&0x8000 > 0 { @@ -119,24 +196,127 @@ func (self *Table) tagToRecord(tag *ESENT_LEAF_ENTRY) *ordereddict.Dict { itemLen = prevItemLen result.Set(column, nil) } else { - data := ParseString(tag.Reader, - variableSizeOffset+variableDataBytesProcessed, - itemLen-prevItemLen) - - switch catalog.Column().ColumnType().Name { - case "Text", "Binary": - result.Set(column, data) + switch column_type { + case "Binary": + result.Set(column, ParseString(tag.Reader, + variableSizeOffset+variableDataBytesProcessed, + itemLen-prevItemLen)) + + case "Text": + result.Set(column, ParseString( + tag.Reader, + variableSizeOffset+variableDataBytesProcessed, + itemLen-prevItemLen)) + + default: + fmt.Printf("Can not handle Column %v variable data %v\n", + column, catalog.Column().DebugString()) } } + if Debug { + fmt.Printf("Consumed %#x bytes of VARIABLE space from %#x\n", + itemLen-prevItemLen, variableDataBytesProcessed) + } + variableDataBytesProcessed += itemLen - prevItemLen prevItemLen = itemLen + + // Tagged values + } else if identifier > 255 { + if taggedItems == nil { + taggedItems = ParseTaggedValues( + self.ctx, getSlice(value.Buffer, + uint64(variableDataBytesProcessed+ + variableSizeOffset), + uint64(len(value.Buffer)))) + } + + buf, pres := taggedItems[identifier] + if pres { + switch column_type { + case "Binary": + result.Set(column, buf) + + case "Long Text": + result.Set(column, ParseTerminatedUTF16String( + &BufferReaderAt{buf}, 0)) + } + } } } return result } +func (self *RecordTag) FlagSkip() uint64 { + return 1 +} + +func getSlice(buffer []byte, start, end uint64) []byte { + if end < start { + return nil + } + + length := uint64(len(buffer)) + + if start < 0 { + start = 0 + } + + if start > length { + start = length + } + + if end > length { + end = length + } + + return buffer[start:end] +} + +func ParseTaggedValues(ctx *ESEContext, buffer []byte) map[uint32][]byte { + result := make(map[uint32][]byte) + + if len(buffer) < 2 { + return result + } + + reader := &BufferReaderAt{buffer} + first_record := ctx.Profile.RecordTag(reader, 0) + prev_record := first_record + + // Iterate over all tag headers - the headers go until the + // start of the first data segment + for offset := uint64(first_record.Size()); offset < first_record.DataOffset(); offset += uint64(first_record.Size()) { + record := ctx.Profile.RecordTag(reader, int64(offset)) + result[uint32(prev_record.Identifier())] = getSlice(buffer, prev_record.DataOffset()+ + prev_record.FlagSkip(), record.DataOffset()) + + if Debug { + fmt.Printf("Consumed %#x bytes of TAGGED space from %#x for tag %#x\n", + record.DataOffset()-prev_record.DataOffset()-prev_record.FlagSkip(), + prev_record.DataOffset()+prev_record.FlagSkip(), + prev_record.Identifier()) + } + + prev_record = record + } + + // Last record goes to the end of the buffer. + result[uint32(prev_record.Identifier())] = getSlice(buffer, prev_record.DataOffset()+ + prev_record.FlagSkip(), uint64(len(buffer))) + + if Debug { + fmt.Printf("Consumed %#x bytes of TAGGED space from %#x for tag %#x\n", + uint64(len(buffer))-prev_record.DataOffset()-prev_record.FlagSkip(), + prev_record.DataOffset()+prev_record.FlagSkip(), + prev_record.Identifier()) + } + + return result +} + // DumpTable extracts all rows in the named table and passes them into // the callback. The callback may cancel the walk at any time by // returning an error which is passed to our caller. @@ -149,8 +329,9 @@ func (self *Catalog) DumpTable(name string, cb func(row *ordereddict.Dict) error table := table_any.(*Table) err := WalkPages(self.ctx, int64(table.FatherDataPageNumber), func(header *PageHeader, id int64, value *Value) error { - leaf_entry := NewESENT_LEAF_ENTRY(self.ctx, value) - return cb(table.tagToRecord(leaf_entry)) + // Each tag stores a single row - all the + // columns in the row are encoded in this tag. + return cb(table.tagToRecord(value)) }) if err != nil { return err @@ -189,7 +370,10 @@ func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) error leaf_entry.Reader, leaf_entry.EntryData()) itemName := parseItemName(dd_header) - catalog := dd_header.Catalog() + + // Catalog follows the dd header + catalog := self.ctx.Profile.ESENT_CATALOG_DATA_DEFINITION_ENTRY(dd_header.Reader, + dd_header.Offset+int64(dd_header.Size())) switch catalog.Type().Name { case "CATALOG_TYPE_TABLE": @@ -205,9 +389,16 @@ func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) error self.Tables.Set(itemName, table) case "CATALOG_TYPE_COLUMN": + if self.currentTable == nil { + return errors.New("Internal Error: No existing table when adding column") + } self.currentTable.Columns.Set(itemName, catalog) case "CATALOG_TYPE_INDEX": + if self.currentTable == nil { + return errors.New("Internal Error: No existing table when adding index") + } + self.currentTable.Indexes.Set(itemName, catalog) case "CATALOG_TYPE_LONG_VALUE": @@ -224,7 +415,8 @@ func (self *Catalog) Dump() string { table := table_any.(*Table) space := " " - result += fmt.Sprintf("[%v]:\n%sColumns\n", table.Name, space) + result += fmt.Sprintf("[%v] (FDP %#x):\n%sColumns\n", table.Name, + table.FatherDataPageNumber, space) for idx, column := range table.Columns.Keys() { catalog_any, _ := table.Columns.Get(column) catalog := catalog_any.(*ESENT_CATALOG_DATA_DEFINITION_ENTRY) diff --git a/context.go b/context.go index 3e46b77..19b642c 100644 --- a/context.go +++ b/context.go @@ -1,27 +1,43 @@ package parser import ( + "errors" + "fmt" "io" ) type ESEContext struct { Reader io.ReaderAt - FileSize int64 Profile *ESEProfile PageSize int64 Header *FileHeader + Version uint32 + Revision uint32 } -func NewESEContext(reader io.ReaderAt, filesize int64) (*ESEContext, error) { +func NewESEContext(reader io.ReaderAt) (*ESEContext, error) { result := &ESEContext{ Profile: NewESEProfile(), Reader: reader, } - // TODO error check. result.Header = result.Profile.FileHeader(reader, 0) + if result.Header.Magic() != 0x89abcdef { + return nil, errors.New(fmt.Sprintf( + "Unsupported ESE file: Magic is %x should be 0x89abcdef", + result.Header.Magic())) + } + result.PageSize = int64(result.Header.PageSize()) + switch result.PageSize { + case 0x1000, 0x2000, 0x4000, 0x8000: + default: + return nil, errors.New(fmt.Sprintf( + "Unsupported page size %x", result.PageSize)) + } + result.Version = result.Header.FormatVersion() + result.Revision = result.Header.FormatRevision() return result, nil } diff --git a/conversion.spec.yaml b/conversion.spec.yaml index 6f160f4..b17452d 100644 --- a/conversion.spec.yaml +++ b/conversion.spec.yaml @@ -24,4 +24,7 @@ Structs: - CATALOG_TYPE_LONG_VALUE - ESENT_DATA_DEFINITION_HEADER - ESENT_CATALOG_DATA_DEFINITION_ENTRY + - RecordTag + + # This one is here to include some utility functions we might need. - Misc diff --git a/debug.go b/debug.go new file mode 100644 index 0000000..e37ad9e --- /dev/null +++ b/debug.go @@ -0,0 +1,9 @@ +package parser + +var ( + // General purpose debug statements. + Debug = false + + // Debugging during walk + DebugWalk = false +) diff --git a/ese_gen.go b/ese_gen.go index 0661e30..755b9e4 100644 --- a/ese_gen.go +++ b/ese_gen.go @@ -64,11 +64,9 @@ type ESEProfile struct { Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Table int64 Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Index int64 Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_LongValue int64 - Off_ESENT_DATA_DEFINITION_HEADER_LastFixedSize int64 + Off_ESENT_DATA_DEFINITION_HEADER_LastFixedType int64 Off_ESENT_DATA_DEFINITION_HEADER_LastVariableDataType int64 Off_ESENT_DATA_DEFINITION_HEADER_VariableSizeOffset int64 - Off_ESENT_DATA_DEFINITION_HEADER_Catalog int64 - Off_ESENT_DATA_DEFINITION_HEADER_FixedSizeStart int64 Off_ESENT_INDEX_ENTRY_RecordPageKey int64 Off_ESENT_LEAF_ENTRY_CommonPageKeySize int64 Off_ESENT_LEAF_ENTRY_LocalPageKeySize int64 @@ -98,6 +96,8 @@ type ESEProfile struct { Off_Misc_Misc int64 Off_Misc_Misc2 int64 Off_Misc_Misc3 int64 + Off_Misc_Misc5 int64 + Off_Misc_Misc4 int64 Off_PageHeader_LastModified int64 Off_PageHeader_PreviousPageNumber int64 Off_PageHeader_NextPageNumber int64 @@ -106,14 +106,15 @@ type ESEProfile struct { Off_PageHeader_AvailableDataOffset int64 Off_PageHeader_AvailablePageTag int64 Off_PageHeader_Flags int64 - Off_Tag_ValueSize int64 - Off_Tag_Flags int64 - Off_Tag_ValueOffset int64 + Off_RecordTag_Identifier int64 + Off_RecordTag_DataOffset int64 + Off_Tag__ValueSize int64 + Off_Tag__ValueOffset int64 } func NewESEProfile() *ESEProfile { // Specific offsets can be tweaked to cater for slight version mismatches. - self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,4,4,0,-2,0,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,1,2,3,4,5,4,12,0,0,0,8,16,20,24,28,32,34,36,0,2,2} + self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,0,-2,0,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,1,2,3,4,5,4,12,0,0,0,0,0,8,16,20,24,28,32,34,36,0,2,0,2} return self } @@ -197,6 +198,10 @@ func (self *ESEProfile) PageHeader(reader io.ReaderAt, offset int64) *PageHeader return &PageHeader{Reader: reader, Offset: offset, Profile: self} } +func (self *ESEProfile) RecordTag(reader io.ReaderAt, offset int64) *RecordTag { + return &RecordTag{Reader: reader, Offset: offset, Profile: self} +} + func (self *ESEProfile) Tag(reader io.ReaderAt, offset int64) *Tag { return &Tag{Reader: reader, Offset: offset, Profile: self} } @@ -531,11 +536,11 @@ type ESENT_DATA_DEFINITION_HEADER struct { } func (self *ESENT_DATA_DEFINITION_HEADER) Size() int { - return 0 + return 4 } -func (self *ESENT_DATA_DEFINITION_HEADER) LastFixedSize() int8 { - return ParseInt8(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_LastFixedSize + self.Offset) +func (self *ESENT_DATA_DEFINITION_HEADER) LastFixedType() int8 { + return ParseInt8(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_LastFixedType + self.Offset) } func (self *ESENT_DATA_DEFINITION_HEADER) LastVariableDataType() byte { @@ -545,21 +550,11 @@ func (self *ESENT_DATA_DEFINITION_HEADER) LastVariableDataType() byte { func (self *ESENT_DATA_DEFINITION_HEADER) VariableSizeOffset() uint16 { return ParseUint16(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_VariableSizeOffset + self.Offset) } - -func (self *ESENT_DATA_DEFINITION_HEADER) Catalog() *ESENT_CATALOG_DATA_DEFINITION_ENTRY { - return self.Profile.ESENT_CATALOG_DATA_DEFINITION_ENTRY(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_Catalog + self.Offset) -} - -func (self *ESENT_DATA_DEFINITION_HEADER) FixedSizeStart() uint64 { - return ParseUint64(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_FixedSizeStart + self.Offset) -} func (self *ESENT_DATA_DEFINITION_HEADER) DebugString() string { result := fmt.Sprintf("struct ESENT_DATA_DEFINITION_HEADER @ %#x:\n", self.Offset) - result += fmt.Sprintf(" LastFixedSize: %#0x\n", self.LastFixedSize()) + result += fmt.Sprintf(" LastFixedType: %#0x\n", self.LastFixedType()) result += fmt.Sprintf(" LastVariableDataType: %#0x\n", self.LastVariableDataType()) result += fmt.Sprintf(" VariableSizeOffset: %#0x\n", self.VariableSizeOffset()) - result += fmt.Sprintf(" Catalog: {\n%v}\n", indent(self.Catalog().DebugString())) - result += fmt.Sprintf(" FixedSizeStart: %#0x\n", self.FixedSizeStart()) return result } @@ -865,11 +860,22 @@ func (self *Misc) Misc2() int16 { func (self *Misc) Misc3() int64 { return int64(ParseUint64(self.Reader, self.Profile.Off_Misc_Misc3 + self.Offset)) } + +func (self *Misc) Misc5() uint64 { + return ParseUint64(self.Reader, self.Profile.Off_Misc_Misc5 + self.Offset) +} + + +func (self *Misc) Misc4() string { + return ParseTerminatedUTF16String(self.Reader, self.Profile.Off_Misc_Misc4 + self.Offset) +} func (self *Misc) DebugString() string { result := fmt.Sprintf("struct Misc @ %#x:\n", self.Offset) result += fmt.Sprintf(" Misc: %#0x\n", self.Misc()) result += fmt.Sprintf(" Misc2: %#0x\n", self.Misc2()) result += fmt.Sprintf(" Misc3: %#0x\n", self.Misc3()) + result += fmt.Sprintf(" Misc5: %#0x\n", self.Misc5()) + result += fmt.Sprintf(" Misc4: %v\n", string(self.Misc4())) return result } @@ -960,6 +966,31 @@ func (self *PageHeader) DebugString() string { return result } +type RecordTag struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *RecordTag) Size() int { + return 4 +} + +func (self *RecordTag) Identifier() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_RecordTag_Identifier + self.Offset) +} + +func (self *RecordTag) DataOffset() uint64 { + value := ParseUint16(self.Reader, self.Profile.Off_RecordTag_DataOffset + self.Offset) + return (uint64(value) & 0x7fff) >> 0x0 +} +func (self *RecordTag) DebugString() string { + result := fmt.Sprintf("struct RecordTag @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Identifier: %#0x\n", self.Identifier()) + result += fmt.Sprintf(" DataOffset: %#0x\n", self.DataOffset()) + return result +} + type Tag struct { Reader io.ReaderAt Offset int64 @@ -970,25 +1001,17 @@ func (self *Tag) Size() int { return 8 } -func (self *Tag) ValueSize() uint64 { - value := ParseUint16(self.Reader, self.Profile.Off_Tag_ValueSize + self.Offset) - return (uint64(value) & 0x1fff) >> 0x0 +func (self *Tag) _ValueSize() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_Tag__ValueSize + self.Offset) } -func (self *Tag) Flags() uint64 { - value := ParseUint16(self.Reader, self.Profile.Off_Tag_Flags + self.Offset) - return (uint64(value) & 0x1fffffff) >> 0xd -} - -func (self *Tag) ValueOffset() uint64 { - value := ParseUint16(self.Reader, self.Profile.Off_Tag_ValueOffset + self.Offset) - return (uint64(value) & 0x1fff) >> 0x0 +func (self *Tag) _ValueOffset() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_Tag__ValueOffset + self.Offset) } func (self *Tag) DebugString() string { result := fmt.Sprintf("struct Tag @ %#x:\n", self.Offset) - result += fmt.Sprintf(" ValueSize: %#0x\n", self.ValueSize()) - result += fmt.Sprintf(" Flags: %#0x\n", self.Flags()) - result += fmt.Sprintf(" ValueOffset: %#0x\n", self.ValueOffset()) + result += fmt.Sprintf(" _ValueSize: %#0x\n", self._ValueSize()) + result += fmt.Sprintf(" _ValueOffset: %#0x\n", self._ValueOffset()) return result } @@ -1119,3 +1142,52 @@ func ParseString(reader io.ReaderAt, offset int64, length int64) string { } +func ParseTerminatedUTF16String(reader io.ReaderAt, offset int64) string { + data := make([]byte, 1024) + n, err := reader.ReadAt(data, offset) + if err != nil && err != io.EOF { + return "" + } + + idx := bytes.Index(data[:n], []byte{0, 0}) + if idx < 0 { + idx = n-1 + } + return UTF16BytesToUTF8(data[0:idx+1], binary.LittleEndian) +} + +func ParseUTF16String(reader io.ReaderAt, offset int64, length int64) string { + data := make([]byte, length) + n, err := reader.ReadAt(data, offset) + if err != nil && err != io.EOF { + return "" + } + return UTF16BytesToUTF8(data[:n], binary.LittleEndian) +} + +func UTF16BytesToUTF8(b []byte, o binary.ByteOrder) string { + if len(b) < 2 { + return "" + } + + if b[0] == 0xff && b[1] == 0xfe { + o = binary.BigEndian + b = b[2:] + } else if b[0] == 0xfe && b[1] == 0xff { + o = binary.LittleEndian + b = b[2:] + } + + utf := make([]uint16, (len(b)+(2-1))/2) + + for i := 0; i+(2-1) < len(b); i += 2 { + utf[i/2] = o.Uint16(b[i:]) + } + if len(b)/2 < len(utf) { + utf[len(utf)-1] = utf8.RuneError + } + + return string(utf16.Decode(utf)) +} + + diff --git a/ese_profile.json b/ese_profile.json index a616439..630d83f 100644 --- a/ese_profile.json +++ b/ese_profile.json @@ -53,21 +53,8 @@ }]] }], "Tag": [8, { - "ValueSize": [0, ["BitField", { - "target": "unsigned short", - "start_bit": 0, - "end_bit": 13 - }]], - "Flags": [2, ["BitField", { - "target": "unsigned short", - "start_bit": 13, - "end_bit": 16 - }]], - "ValueOffset": [2, ["BitField", { - "target": "unsigned short", - "start_bit": 0, - "end_bit": 13 - }]] + "_ValueSize": [0, ["unsigned short"]], + "_ValueOffset": [2, ["unsigned short"]] }], "ESENT_ROOT_HEADER": [16, { "InitialNumberOfPages": [0, ["unsigned long"]], @@ -168,12 +155,10 @@ "InitialNumberOfPages": [12, ["unsigned long"]] }], - "ESENT_DATA_DEFINITION_HEADER": [0, { - "LastFixedSize": [0, ["char"]], + "ESENT_DATA_DEFINITION_HEADER": [4, { + "LastFixedType": [0, ["char"]], "LastVariableDataType": [1, ["unsigned char"]], - "VariableSizeOffset": [2, ["unsigned short"]], - "Catalog": [4, ["ESENT_CATALOG_DATA_DEFINITION_ENTRY"]], - "FixedSizeStart": [4, ["unsigned long long"]] + "VariableSizeOffset": [2, ["unsigned short"]] }], "ESENT_CATALOG_DATA_DEFINITION_ENTRY": [0, { @@ -194,9 +179,20 @@ "LongValue": [10, ["CATALOG_TYPE_LONG_VALUE"]] }], + "RecordTag": [4, { + "Identifier": [0, ["unsigned short"]], + "DataOffset": [2, ["BitField", { + "target": "unsigned short", + "start_bit": 0, + "end_bit": 15 + }]] + }], + "Misc": [0, { "Misc": [0, ["long"]], "Misc2": [0, ["short"]], - "Misc3": [0, ["long long"]] + "Misc3": [0, ["long long"]], + "Misc5": [0, ["unsigned long long"]], + "Misc4": [0, ["UnicodeString"]] }] } diff --git a/pages.go b/pages.go index 055e6f5..d9e9c76 100644 --- a/pages.go +++ b/pages.go @@ -11,14 +11,41 @@ const ( type Value struct { Tag *Tag + PageID int64 Buffer []byte + Flags uint64 } func (self *Value) Reader() io.ReaderAt { return &BufferReaderAt{self.Buffer} } -func GetPageValues(ctx *ESEContext, header *PageHeader) []*Value { +func NewValue(ctx *ESEContext, tag *Tag, PageID int64, buffer []byte) *Value { + result := &Value{Tag: tag, PageID: PageID, Buffer: buffer} + if ctx.Version == 0x620 && ctx.Revision >= 17 && ctx.PageSize > 8192 { + result.Flags = uint64(buffer[1] >> 5) + buffer[1] &= 0x1f + } else { + result.Flags = uint64(tag._ValueOffset()) >> 13 + } + return result +} + +func (self *Tag) ValueOffset(ctx *ESEContext) uint16 { + if ctx.Version == 0x620 && ctx.Revision >= 17 && ctx.PageSize > 8192 { + return self._ValueOffset() & 0x7FFF + } + return self._ValueOffset() & 0x1FFF +} + +func (self *Tag) ValueSize(ctx *ESEContext) uint16 { + if ctx.Version == 0x620 && ctx.Revision >= 17 && ctx.PageSize > 8192 { + return self._ValueSize() & 0x7FFF + } + return self._ValueSize() & 0x1FFF +} + +func GetPageValues(ctx *ESEContext, header *PageHeader, id int64) []*Value { result := []*Value{} // Tags are written from the end of the page @@ -26,15 +53,18 @@ func GetPageValues(ctx *ESEContext, header *PageHeader) []*Value { for tag_count := header.AvailablePageTag(); tag_count > 0; tag_count-- { tag := ctx.Profile.Tag(ctx.Reader, offset) - value_offset := header.Offset + 40 + int64(tag.ValueOffset()) + value_offset := header.EndOffset(ctx) + int64(tag.ValueOffset(ctx)) - buffer := make([]byte, int(tag.ValueSize())) + buffer := make([]byte, int(tag.ValueSize(ctx))) ctx.Reader.ReadAt(buffer, value_offset) - result = append(result, &Value{Tag: tag, Buffer: buffer}) + result = append(result, NewValue(ctx, tag, id, buffer)) offset -= 4 } + if DebugWalk { + fmt.Printf("Got %v values for page %v\n", len(result), id) + } return result } @@ -54,18 +84,33 @@ func (self *PageHeader) IsLeaf() bool { return self.Flags().IsSet("Leaf") } +func (self *PageHeader) EndOffset(ctx *ESEContext) int64 { + // Common size + size := int64(40) + + // Depending on version, the size of the header is different. + if ctx.Version == 0x620 && ctx.Revision >= 0x11 && ctx.PageSize > 8192 { + // Windows 7 and later + size += 5 * 8 + } + + return self.Offset + size +} + func DumpPage(ctx *ESEContext, id int64) { header := ctx.GetPage(id) fmt.Printf("Page %v: %v\n", id, header.DebugString()) // Show the tags - values := GetPageValues(ctx, header) + values := GetPageValues(ctx, header, id) if len(values) == 0 { return } - for _, value := range values { - fmt.Println(value.Tag.DebugString()) + for i, value := range values { + fmt.Printf("Tag %v @ %#x offset %#x length %#x\n", + i, value.Tag.Offset, value.Tag.ValueOffset(ctx), + value.Tag.ValueSize(ctx)) } flags := header.Flags() @@ -120,7 +165,7 @@ func (self *ESENT_INDEX_ENTRY) Dump() { // CommonPageKeySize field before the struct. This constructor then // positions the struct appropriately. func NewESENT_LEAF_ENTRY(ctx *ESEContext, value *Value) *ESENT_LEAF_ENTRY { - if value.Tag.Flags()&TAG_COMMON > 0 { + if value.Flags&TAG_COMMON > 0 { // Skip the common header return ctx.Profile.ESENT_LEAF_ENTRY(value.Reader(), 2) } @@ -145,7 +190,7 @@ func (self *ESENT_BRANCH_HEADER) Dump() { // CommonPageKeySize field before the struct. This construstor then // positions the struct appropriately. func NewESENT_BRANCH_ENTRY(ctx *ESEContext, value *Value) *ESENT_BRANCH_ENTRY { - if value.Tag.Flags()&TAG_COMMON > 0 { + if value.Flags&TAG_COMMON > 0 { // Skip the common header return ctx.Profile.ESENT_BRANCH_ENTRY(value.Reader(), 2) } @@ -181,8 +226,12 @@ func WalkPages(ctx *ESEContext, return nil } + if DebugWalk { + fmt.Printf("Walking page %v\n", id) + } + header := ctx.GetPage(id) - values := GetPageValues(ctx, header) + values := GetPageValues(ctx, header, id) // No more records. if len(values) == 0 { diff --git a/reader.go b/reader.go index 080c8aa..c2444ee 100644 --- a/reader.go +++ b/reader.go @@ -6,6 +6,11 @@ type BufferReaderAt struct { func (self *BufferReaderAt) ReadAt(buf []byte, offset int64) (int, error) { to_read := int64(len(buf)) + if offset < 0 { + to_read += offset + offset = 0 + } + if offset+to_read > int64(len(self.buffer)) { to_read = int64(len(self.buffer)) - offset } From 454d6d7f1b0da73170577757936a5e83669f5087 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Tue, 3 Dec 2019 01:17:37 +1000 Subject: [PATCH 10/40] Fixed bug in parsing tagged values. (#3) --- catalog.go | 11 +++++++++-- ese_gen.go | 26 +++++++++++++------------- ese_profile.json | 2 +- 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/catalog.go b/catalog.go index 39f9d19..8febb63 100644 --- a/catalog.go +++ b/catalog.go @@ -3,6 +3,7 @@ package parser import ( + "encoding/hex" "errors" "fmt" "math" @@ -235,12 +236,18 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { buf, pres := taggedItems[identifier] if pres { switch column_type { - case "Binary": - result.Set(column, buf) + case "Binary", "Long Binary": + result.Set(column, hex.EncodeToString(buf)) case "Long Text": result.Set(column, ParseTerminatedUTF16String( &BufferReaderAt{buf}, 0)) + + default: + if Debug { + fmt.Printf("Can not handle Column %v tagged data %v\n", + column, catalog.Column().DebugString()) + } } } } diff --git a/ese_gen.go b/ese_gen.go index 755b9e4..4cf606e 100644 --- a/ese_gen.go +++ b/ese_gen.go @@ -922,18 +922,6 @@ func (self *PageHeader) Flags() *Flags { names := make(map[string]bool) - if value & 8 != 0 { - names["Empty"] = true - } - - if value & 32 != 0 { - names["SpaceTree"] = true - } - - if value & 64 != 0 { - names["Index"] = true - } - if value & 128 != 0 { names["Long"] = true } @@ -950,6 +938,18 @@ func (self *PageHeader) Flags() *Flags { names["Parent"] = true } + if value & 8 != 0 { + names["Empty"] = true + } + + if value & 32 != 0 { + names["SpaceTree"] = true + } + + if value & 64 != 0 { + names["Index"] = true + } + return &Flags{Value: uint64(value), Names: names} } @@ -982,7 +982,7 @@ func (self *RecordTag) Identifier() uint16 { func (self *RecordTag) DataOffset() uint64 { value := ParseUint16(self.Reader, self.Profile.Off_RecordTag_DataOffset + self.Offset) - return (uint64(value) & 0x7fff) >> 0x0 + return (uint64(value) & 0x1fff) >> 0x0 } func (self *RecordTag) DebugString() string { result := fmt.Sprintf("struct RecordTag @ %#x:\n", self.Offset) diff --git a/ese_profile.json b/ese_profile.json index 630d83f..3cf1d38 100644 --- a/ese_profile.json +++ b/ese_profile.json @@ -184,7 +184,7 @@ "DataOffset": [2, ["BitField", { "target": "unsigned short", "start_bit": 0, - "end_bit": 15 + "end_bit": 13 }]] }], From fbe615aab0dca36a30d9f8300a93e839a0cffbad Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Tue, 3 Dec 2019 09:52:49 +1000 Subject: [PATCH 11/40] Performance optimizations. (#4) --- catalog.go | 119 ++++++++++++++++++++++++++++------------------------- 1 file changed, 63 insertions(+), 56 deletions(-) diff --git a/catalog.go b/catalog.go index 8febb63..a600bee 100644 --- a/catalog.go +++ b/catalog.go @@ -17,12 +17,21 @@ const ( CATALOG_PAGE_NUMBER = 4 ) +// Store a simple struct of column spec for speed. +type ColumnSpec struct { + FDPId uint32 + Name string + Identifier uint32 + Type string + SpaceUsage int64 +} + type Table struct { ctx *ESEContext Header *CATALOG_TYPE_TABLE FatherDataPageNumber uint32 Name string - Columns *ordereddict.Dict + Columns []*ColumnSpec Indexes *ordereddict.Dict LongValues *ordereddict.Dict } @@ -94,124 +103,119 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { prevItemLen := int64(0) variableSizeOffset := dd_header.Offset + int64(dd_header.VariableSizeOffset()) variableDataBytesProcessed := int64(dd_header.LastVariableDataType()-127) * 2 + last_fixed_type := uint32(dd_header.LastFixedType()) + last_variable_data_type := uint32(dd_header.LastVariableDataType()) // Iterate over the column definitions and decode each // identifier according to where it comes from. - for _, column := range self.Columns.Keys() { - catalog_any, _ := self.Columns.Get(column) - catalog := catalog_any.(*ESENT_CATALOG_DATA_DEFINITION_ENTRY) - identifier := catalog.Identifier() - column_type := catalog.Column().ColumnType().Name - + for _, column := range self.Columns { if Debug { - fmt.Printf("Column %v Identifier %v Type %v\n", column, identifier, - catalog.Column().ColumnType().Name) + fmt.Printf("Column %v Identifier %v Type %v\n", column.Name, + column.Identifier, column.Type) } // Column is stored in the fixed section. - if identifier <= uint32(dd_header.LastFixedType()) { - space_usage := catalog.Column().SpaceUsage() - - switch column_type { + if column.Identifier <= last_fixed_type { + switch column.Type { case "Boolean": - if space_usage == 1 { - result.Set(column, ParseUint8(tag.Reader, offset) > 0) + if column.SpaceUsage == 1 { + result.Set(column.Name, ParseUint8(tag.Reader, offset) > 0) } case "Signed byte": - if space_usage == 1 { - result.Set(column, ParseUint8(tag.Reader, offset)) + if column.SpaceUsage == 1 { + result.Set(column.Name, ParseUint8(tag.Reader, offset)) } case "Signed short": - if space_usage == 2 { - result.Set(column, ParseInt16(tag.Reader, offset)) + if column.SpaceUsage == 2 { + result.Set(column.Name, ParseInt16(tag.Reader, offset)) } case "Unsigned short": - if space_usage == 2 { - result.Set(column, ParseUint16(tag.Reader, offset)) + if column.SpaceUsage == 2 { + result.Set(column.Name, ParseUint16(tag.Reader, offset)) } case "Signed long": - if space_usage == 4 { - result.Set(column, ParseInt32(tag.Reader, offset)) + if column.SpaceUsage == 4 { + result.Set(column.Name, ParseInt32(tag.Reader, offset)) } case "Unsigned long": - if space_usage == 4 { - result.Set(column, ParseUint32(tag.Reader, offset)) + if column.SpaceUsage == 4 { + result.Set(column.Name, ParseUint32(tag.Reader, offset)) } case "Single precision FP": - if space_usage == 4 { - result.Set(column, math.Float32frombits( + if column.SpaceUsage == 4 { + result.Set(column.Name, math.Float32frombits( ParseUint32(tag.Reader, offset))) } case "Double precision FP": - if space_usage == 8 { - result.Set(column, math.Float64frombits( + if column.SpaceUsage == 8 { + result.Set(column.Name, math.Float64frombits( ParseUint64(tag.Reader, offset))) } case "DateTime": - if space_usage == 8 { + if column.SpaceUsage == 8 { // Some hair brained time serialization method // https://docs.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-coltyp days_since_1900 := math.Float64frombits( ParseUint64(tag.Reader, offset)) // In python time.mktime((1900,1,1,0,0,0,0,365,0)) - result.Set(column, time.Unix(int64(days_since_1900*24*60*60)+ + result.Set(column.Name, time.Unix(int64(days_since_1900*24*60*60)+ -2208988800, 0)) } case "Long long", "Currency": - if space_usage == 8 { - result.Set(column, ParseUint64(tag.Reader, offset)) + if column.SpaceUsage == 8 { + result.Set(column.Name, ParseUint64(tag.Reader, offset)) } default: fmt.Printf("Can not handle Column %v fixed data %v\n", - column, catalog.Column().DebugString()) + column.Name, column) } if Debug { fmt.Printf("Consumed %#x bytes of FIXED space from %#x\n", - catalog.Column().SpaceUsage(), offset) + column.SpaceUsage, offset) } // Move our offset along - offset += int64(catalog.Column().SpaceUsage()) + offset += column.SpaceUsage // Identifier is stored in the variable section - } else if 127 < identifier && - identifier <= uint32(dd_header.LastVariableDataType()) { + } else if 127 < column.Identifier && + column.Identifier <= last_variable_data_type { // Variable data type - index := int64(identifier) - 127 - 1 + index := int64(column.Identifier) - 127 - 1 itemLen := int64(ParseUint16(tag.Reader, variableSizeOffset+index*2)) if itemLen&0x8000 > 0 { // Empty Item itemLen = prevItemLen - result.Set(column, nil) + result.Set(column.Name, nil) } else { - switch column_type { + switch column.Type { case "Binary": - result.Set(column, ParseString(tag.Reader, + result.Set(column.Name, ParseString(tag.Reader, variableSizeOffset+variableDataBytesProcessed, itemLen-prevItemLen)) case "Text": - result.Set(column, ParseString( + result.Set(column.Name, ParseString( tag.Reader, variableSizeOffset+variableDataBytesProcessed, itemLen-prevItemLen)) default: fmt.Printf("Can not handle Column %v variable data %v\n", - column, catalog.Column().DebugString()) + column.Name, column) } } @@ -224,7 +228,7 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { prevItemLen = itemLen // Tagged values - } else if identifier > 255 { + } else if column.Identifier > 255 { if taggedItems == nil { taggedItems = ParseTaggedValues( self.ctx, getSlice(value.Buffer, @@ -233,20 +237,20 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { uint64(len(value.Buffer)))) } - buf, pres := taggedItems[identifier] + buf, pres := taggedItems[column.Identifier] if pres { - switch column_type { + switch column.Type { case "Binary", "Long Binary": - result.Set(column, hex.EncodeToString(buf)) + result.Set(column.Name, hex.EncodeToString(buf)) case "Long Text": - result.Set(column, ParseTerminatedUTF16String( + result.Set(column.Name, ParseTerminatedUTF16String( &BufferReaderAt{buf}, 0)) default: if Debug { fmt.Printf("Can not handle Column %v tagged data %v\n", - column, catalog.Column().DebugString()) + column.Name, column) } } } @@ -389,7 +393,6 @@ func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) error Header: catalog.Table(), Name: itemName, FatherDataPageNumber: catalog.Table().FatherDataPageNumber(), - Columns: ordereddict.NewDict(), Indexes: ordereddict.NewDict(), LongValues: ordereddict.NewDict()} self.currentTable = table @@ -399,7 +402,13 @@ func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) error if self.currentTable == nil { return errors.New("Internal Error: No existing table when adding column") } - self.currentTable.Columns.Set(itemName, catalog) + self.currentTable.Columns = append(self.currentTable.Columns, &ColumnSpec{ + Name: itemName, + FDPId: catalog.FDPId(), + Identifier: catalog.Identifier(), + Type: catalog.Column().ColumnType().Name, + SpaceUsage: int64(catalog.Column().SpaceUsage()), + }) case "CATALOG_TYPE_INDEX": if self.currentTable == nil { @@ -424,11 +433,9 @@ func (self *Catalog) Dump() string { space := " " result += fmt.Sprintf("[%v] (FDP %#x):\n%sColumns\n", table.Name, table.FatherDataPageNumber, space) - for idx, column := range table.Columns.Keys() { - catalog_any, _ := table.Columns.Get(column) - catalog := catalog_any.(*ESENT_CATALOG_DATA_DEFINITION_ENTRY) + for idx, column := range table.Columns { result += fmt.Sprintf("%s%s%-5d%-30v%v\n", space, space, idx, - column, catalog.Column().ColumnType().Name) + column.Name, column.Type) } result += fmt.Sprintf("%sIndexes\n", space) From 40f3521dd14c7e29c9be5e743975c788b33d1d59 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Tue, 3 Dec 2019 10:28:41 +1000 Subject: [PATCH 12/40] Bugfix: Do not walk pages we walked before. (#5) --- pages.go | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/pages.go b/pages.go index d9e9c76..c999328 100644 --- a/pages.go +++ b/pages.go @@ -222,9 +222,20 @@ func (self *ESENT_LEAF_HEADER) Dump() { func WalkPages(ctx *ESEContext, id int64, cb func(header *PageHeader, page_id int64, value *Value) error) error { - if id <= 0 { + seen := make(map[int64]bool) + + return _walkPages(ctx, id, seen, cb) +} + +func _walkPages(ctx *ESEContext, + id int64, seen map[int64]bool, + cb func(header *PageHeader, page_id int64, value *Value) error) error { + + _, pres := seen[id] + if id <= 0 || pres { return nil } + seen[id] = true if DebugWalk { fmt.Printf("Walking page %v\n", id) @@ -248,7 +259,7 @@ func WalkPages(ctx *ESEContext, } else if header.IsBranch() { // Walk the branch branch := NewESENT_BRANCH_ENTRY(ctx, value) - err := WalkPages(ctx, branch.ChildPageNumber(), cb) + err := _walkPages(ctx, branch.ChildPageNumber(), seen, cb) if err != nil { return err } @@ -256,7 +267,7 @@ func WalkPages(ctx *ESEContext, } if header.NextPageNumber() > 0 { - err := WalkPages(ctx, int64(header.NextPageNumber()), cb) + err := _walkPages(ctx, int64(header.NextPageNumber()), seen, cb) if err != nil { return err } From e7c15030dab621f74d78979589e4c4285189e00a Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sat, 11 Jan 2020 17:01:59 +1000 Subject: [PATCH 13/40] Check for empty buffer (#7) Avoids panic on empty tags --- pages.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pages.go b/pages.go index c999328..1eb86fb 100644 --- a/pages.go +++ b/pages.go @@ -22,7 +22,8 @@ func (self *Value) Reader() io.ReaderAt { func NewValue(ctx *ESEContext, tag *Tag, PageID int64, buffer []byte) *Value { result := &Value{Tag: tag, PageID: PageID, Buffer: buffer} - if ctx.Version == 0x620 && ctx.Revision >= 17 && ctx.PageSize > 8192 { + if ctx.Version == 0x620 && ctx.Revision >= 17 && + ctx.PageSize > 8192 && len(buffer) > 0 { result.Flags = uint64(buffer[1] >> 5) buffer[1] &= 0x1f } else { From 0d78f9f9da539bbf686d783075a0cd07441ad3a8 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Mon, 6 Apr 2020 00:30:47 +1000 Subject: [PATCH 14/40] Force JSON serialization of times in UTC (#2) Due to Golang limitations it is not possible to force json serialization of time.Time into UTC in all cases. This hack is a workaround this limitation by special casing time.Time inside an ordereddict.Dict. It works well in most cases. --- ordereddict.go | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/ordereddict.go b/ordereddict.go index bab2ef2..4ee4418 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -7,6 +7,7 @@ import ( "io" "strings" "sync" + "time" ) // A concerete implementation of a row - similar to Python's @@ -328,7 +329,7 @@ func (self Dict) MarshalJSON() ([]byte, error) { // add value v := self.store[k] - vBytes, err := json.Marshal(v) + vBytes, err := marshal(v) if err == nil { result += string(vBytes) + "," } else { @@ -341,3 +342,18 @@ func (self Dict) MarshalJSON() ([]byte, error) { result = result + "}" return []byte(result), nil } + +func marshal(v interface{}) ([]byte, error) { + switch t := v.(type) { + case time.Time: + // Always marshal times as UTC + return json.Marshal(t.UTC()) + + case *time.Time: + // Always marshal times as UTC + return json.Marshal(t.UTC()) + + default: + return json.Marshal(v) + } +} From 17494970552f4c293371f4f51b9ef2f7eaaaa683 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sat, 18 Apr 2020 18:11:15 +1000 Subject: [PATCH 15/40] Print dict as a JSON string. (#3) This is usually more useful than an ad-hoc String() method. --- ordereddict.go | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/ordereddict.go b/ordereddict.go index 4ee4418..bfda8bb 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -148,19 +148,13 @@ func (self *Dict) ToDict() *map[string]interface{} { return &result } +// Printing the dict will always result in a valid JSON document. func (self *Dict) String() string { - self.Lock() - defer self.Unlock() - - builder := make([]string, len(self.keys)) - - var index int = 0 - for _, key := range self.keys { - val, _ := self.store[key] - builder[index] = fmt.Sprintf("%v:%v, ", key, val) - index++ + serialized, err := self.MarshalJSON() + if err != nil { + return fmt.Sprintf("Error: %v", err) } - return fmt.Sprintf("Dict%v", builder) + return string(serialized) } func (self *Dict) GoString() string { From bc44e4ef79c7ef16d23a666f4ab90022c3f557da Mon Sep 17 00:00:00 2001 From: Michael Cohen Date: Mon, 20 Apr 2020 04:37:04 +1000 Subject: [PATCH 16/40] Added Delete() method This is not very efficient but should be ok for small dicts. --- ordereddict.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ordereddict.go b/ordereddict.go index bfda8bb..b54c948 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -77,6 +77,18 @@ func remove(s []string, r string) []string { return s } +// Very inefficient but ok for occasional use. +func (self *Dict) Delete(key string) { + new_keys := make([]string, 0, len(self.keys)) + for _, old_key := range self.keys { + if key != old_key { + new_keys = append(new_keys, old_key) + } + } + self.keys = new_keys + delete(self.store, key) +} + func (self *Dict) Set(key string, value interface{}) *Dict { self.Lock() defer self.Unlock() From 4cd604876fe5e9b1ced3618b59576cc54cee5e25 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Tue, 28 Apr 2020 17:01:54 +1000 Subject: [PATCH 17/40] Added convenience methods GetString() and GetInt64() (#4) --- ordereddict.go | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/ordereddict.go b/ordereddict.go index b54c948..152cc66 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -137,6 +137,39 @@ func (self *Dict) Get(key string) (interface{}, bool) { return val, ok } +func (self *Dict) GetString(key string) (string, bool) { + v, pres := self.Get(key) + if pres { + v_str, ok := v.(string) + if ok { + return v_str, true + } + } + return "", false +} + +func (self *Dict) GetStrings(key string) ([]string, bool) { + v, pres := self.Get(key) + if pres { + v_str, ok := v.([]string) + if ok { + return v_str, true + } + } + return nil, false +} + +func (self *Dict) GetInt64(key string) (int64, bool) { + v, pres := self.Get(key) + if pres { + v_int, ok := v.(int64) + if ok { + return v_int, true + } + } + return 0, false +} + func (self *Dict) Keys() []string { self.Lock() defer self.Unlock() @@ -317,7 +350,7 @@ func handledelim(token json.Token, dec *json.Decoder) (res interface{}, err erro return token, nil } -func (self Dict) MarshalJSON() ([]byte, error) { +func (self *Dict) MarshalJSON() ([]byte, error) { self.Lock() defer self.Unlock() From b012020161d20a77b420acddb99bfc9caded35e0 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sun, 31 May 2020 21:47:11 +1000 Subject: [PATCH 18/40] Support all int types for GetInt64() (#5) --- ordereddict.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/ordereddict.go b/ordereddict.go index 152cc66..34a1a96 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -160,11 +160,31 @@ func (self *Dict) GetStrings(key string) ([]string, bool) { } func (self *Dict) GetInt64(key string) (int64, bool) { - v, pres := self.Get(key) + value, pres := self.Get(key) if pres { - v_int, ok := v.(int64) - if ok { - return v_int, true + switch t := value.(type) { + case int: + return int64(t), true + case int8: + return int64(t), true + case int16: + return int64(t), true + case int32: + return int64(t), true + case int64: + return int64(t), true + case uint8: + return int64(t), true + case uint16: + return int64(t), true + case uint32: + return int64(t), true + case uint64: + return int64(t), true + case float32: + return int64(t), true + case float64: + return int64(t), true } } return 0, false From 52f2ea5ca30e7007c995cdf421637d1b71d08de3 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Tue, 2 Jun 2020 23:33:21 +1000 Subject: [PATCH 19/40] Fixed bug in getstrings. (#6) --- ordereddict.go | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/ordereddict.go b/ordereddict.go index 34a1a96..d651100 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "reflect" "strings" "sync" "time" @@ -140,7 +141,7 @@ func (self *Dict) Get(key string) (interface{}, bool) { func (self *Dict) GetString(key string) (string, bool) { v, pres := self.Get(key) if pres { - v_str, ok := v.(string) + v_str, ok := to_string(v) if ok { return v_str, true } @@ -148,12 +149,33 @@ func (self *Dict) GetString(key string) (string, bool) { return "", false } +func to_string(x interface{}) (string, bool) { + switch t := x.(type) { + case string: + return t, true + case *string: + return *t, true + case []byte: + return string(t), true + default: + return "", false + } +} + func (self *Dict) GetStrings(key string) ([]string, bool) { v, pres := self.Get(key) if pres { - v_str, ok := v.([]string) - if ok { - return v_str, true + slice := reflect.ValueOf(v) + if slice.Type().Kind() == reflect.Slice { + result := []string{} + for i := 0; i < slice.Len(); i++ { + value := slice.Index(i).Interface() + item, ok := to_string(value) + if ok { + result = append(result, item) + } + } + return result, true } } return nil, false From 9f5b751d8d245d41856ded790c40f0be764777d6 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Thu, 4 Jun 2020 22:28:19 +1000 Subject: [PATCH 20/40] Fixed crash with GetStrings when value is nil (#7) --- ordereddict.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ordereddict.go b/ordereddict.go index d651100..fb73d1c 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -164,7 +164,7 @@ func to_string(x interface{}) (string, bool) { func (self *Dict) GetStrings(key string) ([]string, bool) { v, pres := self.Get(key) - if pres { + if pres && v != nil { slice := reflect.ValueOf(v) if slice.Type().Kind() == reflect.Slice { result := []string{} From 9460a6764ab85c9b2d02899ef259cccc70e464a6 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Fri, 24 Jul 2020 01:35:57 +1000 Subject: [PATCH 21/40] No longer force json encoding of time to UTC (#8) --- ordereddict.go | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/ordereddict.go b/ordereddict.go index fb73d1c..46618ac 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -8,7 +8,6 @@ import ( "reflect" "strings" "sync" - "time" ) // A concerete implementation of a row - similar to Python's @@ -410,7 +409,7 @@ func (self *Dict) MarshalJSON() ([]byte, error) { // add value v := self.store[k] - vBytes, err := marshal(v) + vBytes, err := json.Marshal(v) if err == nil { result += string(vBytes) + "," } else { @@ -423,18 +422,3 @@ func (self *Dict) MarshalJSON() ([]byte, error) { result = result + "}" return []byte(result), nil } - -func marshal(v interface{}) ([]byte, error) { - switch t := v.(type) { - case time.Time: - // Always marshal times as UTC - return json.Marshal(t.UTC()) - - case *time.Time: - // Always marshal times as UTC - return json.Marshal(t.UTC()) - - default: - return json.Marshal(v) - } -} From cf5d9045c0d1f4c98a183f54e91854a261bc19c9 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sun, 2 May 2021 18:23:34 +1000 Subject: [PATCH 22/40] Prevent deadlock when self reference (#9) --- ordereddict.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/ordereddict.go b/ordereddict.go index 46618ac..b92888d 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -89,6 +89,21 @@ func (self *Dict) Delete(key string) { delete(self.store, key) } +// Like Set() but does not effect the order. +func (self *Dict) Update(key string, value interface{}) *Dict { + self.Lock() + defer self.Unlock() + + _, pres := self.store[key] + if pres { + self.store[key] = value + } else { + self.Set(key, value) + } + + return self +} + func (self *Dict) Set(key string, value interface{}) *Dict { self.Lock() defer self.Unlock() @@ -404,11 +419,16 @@ func (self *Dict) MarshalJSON() ([]byte, error) { continue } - result += string(kEscaped) + ":" - // add value v := self.store[k] + // Check for back references and skip them - this is not perfect. + subdict, ok := v.(*Dict) + if ok && subdict == self { + continue + } + + result += string(kEscaped) + ":" vBytes, err := json.Marshal(v) if err == nil { result += string(vBytes) + "," From aa39c370e491640a3eca0bdc4715e75f6d2aa157 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Thu, 14 Oct 2021 12:51:16 +1000 Subject: [PATCH 23/40] Added GUID cell type (#1) (#11) To support new User Access Log type ESE files. --- catalog.go | 47 ++++++++++++++++++++------- conversion.spec.yaml | 1 + ese_gen.go | 77 ++++++++++++++++++++++++++++++++++++++------ ese_profile.json | 10 ++++++ guid.go | 12 +++++++ utils.go | 11 +++++++ 6 files changed, 137 insertions(+), 21 deletions(-) create mode 100644 guid.go create mode 100644 utils.go diff --git a/catalog.go b/catalog.go index a600bee..61c91c4 100644 --- a/catalog.go +++ b/catalog.go @@ -23,6 +23,7 @@ type ColumnSpec struct { Name string Identifier uint32 Type string + Flags uint32 SpaceUsage int64 } @@ -161,20 +162,39 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { case "DateTime": if column.SpaceUsage == 8 { - // Some hair brained time serialization method - // https://docs.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-coltyp - days_since_1900 := math.Float64frombits( - ParseUint64(tag.Reader, offset)) - // In python time.mktime((1900,1,1,0,0,0,0,365,0)) - result.Set(column.Name, time.Unix(int64(days_since_1900*24*60*60)+ - -2208988800, 0)) + switch column.Flags { + case 1: + // A more modern way of encoding + result.Set(column.Name, WinFileTime64(tag.Reader, offset)) + + case 0: + // Some hair brained time serialization method + // https://docs.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-coltyp + + value_int := ParseUint64(tag.Reader, offset) + days_since_1900 := math.Float64frombits(value_int) + + // In python time.mktime((1900,1,1,0,0,0,0,365,0)) + result.Set(column.Name, + time.Unix(int64(days_since_1900*24*60*60)+ + -2208988800, 0).UTC()) + + default: + // We have no idea + result.Set(column.Name, ParseUint64(tag.Reader, offset)) + } } - case "Long long", "Currency": if column.SpaceUsage == 8 { result.Set(column.Name, ParseUint64(tag.Reader, offset)) } + case "GUID": + if column.SpaceUsage == 16 { + result.Set(column.Name, + self.Header.Profile.GUID(tag.Reader, offset).AsString()) + } + default: fmt.Printf("Can not handle Column %v fixed data %v\n", column.Name, column) @@ -402,12 +422,15 @@ func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) error if self.currentTable == nil { return errors.New("Internal Error: No existing table when adding column") } + column := catalog.Column() + self.currentTable.Columns = append(self.currentTable.Columns, &ColumnSpec{ Name: itemName, FDPId: catalog.FDPId(), Identifier: catalog.Identifier(), - Type: catalog.Column().ColumnType().Name, - SpaceUsage: int64(catalog.Column().SpaceUsage()), + Type: column.ColumnType().Name, + Flags: column.ColumnFlags(), + SpaceUsage: int64(column.SpaceUsage()), }) case "CATALOG_TYPE_INDEX": @@ -434,8 +457,8 @@ func (self *Catalog) Dump() string { result += fmt.Sprintf("[%v] (FDP %#x):\n%sColumns\n", table.Name, table.FatherDataPageNumber, space) for idx, column := range table.Columns { - result += fmt.Sprintf("%s%s%-5d%-30v%v\n", space, space, idx, - column.Name, column.Type) + result += fmt.Sprintf("%s%s%-5d%-30v%-15vFlags %v\n", space, space, idx, + column.Name, column.Type, column.Flags) } result += fmt.Sprintf("%sIndexes\n", space) diff --git a/conversion.spec.yaml b/conversion.spec.yaml index b17452d..e2f54d8 100644 --- a/conversion.spec.yaml +++ b/conversion.spec.yaml @@ -25,6 +25,7 @@ Structs: - ESENT_DATA_DEFINITION_HEADER - ESENT_CATALOG_DATA_DEFINITION_ENTRY - RecordTag + - GUID # This one is here to include some utility functions we might need. - Misc diff --git a/ese_gen.go b/ese_gen.go index 4cf606e..cfe56f3 100644 --- a/ese_gen.go +++ b/ese_gen.go @@ -85,6 +85,10 @@ type ESEProfile struct { Off_FileHeader_DataBaseTime int64 Off_FileHeader_Signature int64 Off_FileHeader_PageSize int64 + Off_GUID_Data1 int64 + Off_GUID_Data2 int64 + Off_GUID_Data3 int64 + Off_GUID_Data4 int64 Off_JET_LOGTIME_Sec int64 Off_JET_LOGTIME_Min int64 Off_JET_LOGTIME_Hours int64 @@ -114,7 +118,7 @@ type ESEProfile struct { func NewESEProfile() *ESEProfile { // Specific offsets can be tweaked to cater for slight version mismatches. - self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,0,-2,0,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,1,2,3,4,5,4,12,0,0,0,0,0,8,16,20,24,28,32,34,36,0,2,0,2} + self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,0,-2,0,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,4,6,8,0,1,2,3,4,5,4,12,0,0,0,0,0,8,16,20,24,28,32,34,36,0,2,0,2} return self } @@ -182,6 +186,10 @@ func (self *ESEProfile) FileHeader(reader io.ReaderAt, offset int64) *FileHeader return &FileHeader{Reader: reader, Offset: offset, Profile: self} } +func (self *ESEProfile) GUID(reader io.ReaderAt, offset int64) *GUID { + return &GUID{Reader: reader, Offset: offset, Profile: self} +} + func (self *ESEProfile) JET_LOGTIME(reader io.ReaderAt, offset int64) *JET_LOGTIME { return &JET_LOGTIME{Reader: reader, Offset: offset, Profile: self} } @@ -770,6 +778,39 @@ func (self *FileHeader) DebugString() string { return result } +type GUID struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *GUID) Size() int { + return 16 +} + +func (self *GUID) Data1() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_GUID_Data1 + self.Offset) +} + +func (self *GUID) Data2() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_GUID_Data2 + self.Offset) +} + +func (self *GUID) Data3() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_GUID_Data3 + self.Offset) +} + +func (self *GUID) Data4() []byte { + return ParseArray_byte(self.Profile, self.Reader, self.Profile.Off_GUID_Data4 + self.Offset, 8) +} +func (self *GUID) DebugString() string { + result := fmt.Sprintf("struct GUID @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Data1: %#0x\n", self.Data1()) + result += fmt.Sprintf(" Data2: %#0x\n", self.Data2()) + result += fmt.Sprintf(" Data3: %#0x\n", self.Data3()) + return result +} + type JET_LOGTIME struct { Reader io.ReaderAt Offset int64 @@ -922,14 +963,6 @@ func (self *PageHeader) Flags() *Flags { names := make(map[string]bool) - if value & 128 != 0 { - names["Long"] = true - } - - if value & 1 != 0 { - names["Root"] = true - } - if value & 2 != 0 { names["Leaf"] = true } @@ -950,6 +983,14 @@ func (self *PageHeader) Flags() *Flags { names["Index"] = true } + if value & 128 != 0 { + names["Long"] = true + } + + if value & 1 != 0 { + names["Root"] = true + } + return &Flags{Value: uint64(value), Names: names} } @@ -1046,6 +1087,24 @@ func (self Flags) IsSet(flag string) bool { return result } +func (self Flags) Values() []string { + result := make([]string, 0, len(self.Names)) + for k, _ := range self.Names { + result = append(result, k) + } + return result +} + + +func ParseArray_byte(profile *ESEProfile, reader io.ReaderAt, offset int64, count int) []byte { + result := make([]byte, 0, count) + for i:=0; i Date: Thu, 14 Oct 2021 22:09:51 +1000 Subject: [PATCH 24/40] Support integers in tagged sections (#12) Tagged sections are used to represent sparse rows. This is used to store UAL day columns. --- catalog.go | 168 ++++++++++++++++++++++++++++++++++++++++------- ese_gen.go | 17 +++-- ese_profile.json | 5 ++ 3 files changed, 160 insertions(+), 30 deletions(-) diff --git a/catalog.go b/catalog.go index 61c91c4..05a1b47 100644 --- a/catalog.go +++ b/catalog.go @@ -184,6 +184,7 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { result.Set(column.Name, ParseUint64(tag.Reader, offset)) } } + case "Long long", "Currency": if column.SpaceUsage == 8 { result.Set(column.Name, ParseUint64(tag.Reader, offset)) @@ -220,6 +221,7 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { // Empty Item itemLen = prevItemLen result.Set(column.Name, nil) + } else { switch column.Type { case "Binary": @@ -250,22 +252,109 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { // Tagged values } else if column.Identifier > 255 { if taggedItems == nil { + if Debug { + fmt.Printf("Slice is %#x-%#x %x\n", + variableDataBytesProcessed+variableSizeOffset, + len(value.Buffer), getSlice(value.Buffer, + uint64(variableDataBytesProcessed+ + variableSizeOffset), + uint64(len(value.Buffer)+1))) + } taggedItems = ParseTaggedValues( self.ctx, getSlice(value.Buffer, uint64(variableDataBytesProcessed+ variableSizeOffset), - uint64(len(value.Buffer)))) + uint64(len(value.Buffer)+1))) } buf, pres := taggedItems[column.Identifier] if pres { + reader := &BufferReaderAt{buf} switch column.Type { case "Binary", "Long Binary": result.Set(column.Name, hex.EncodeToString(buf)) case "Long Text": result.Set(column.Name, ParseTerminatedUTF16String( - &BufferReaderAt{buf}, 0)) + reader, 0)) + + case "Boolean": + if column.SpaceUsage == 1 { + result.Set(column.Name, ParseUint8(reader, 0) > 0) + } + + case "Signed byte": + if column.SpaceUsage == 1 { + result.Set(column.Name, ParseUint8(reader, 0)) + } + + case "Signed short": + if column.SpaceUsage == 2 { + result.Set(column.Name, ParseInt16(reader, 0)) + } + + case "Unsigned short": + if column.SpaceUsage == 2 { + result.Set(column.Name, ParseUint16(reader, 0)) + } + + case "Signed long": + if column.SpaceUsage == 4 { + result.Set(column.Name, ParseInt32(reader, 0)) + } + + case "Unsigned long": + if column.SpaceUsage == 4 { + result.Set(column.Name, ParseUint32(reader, 0)) + } + + case "Single precision FP": + if column.SpaceUsage == 4 { + result.Set(column.Name, math.Float32frombits( + ParseUint32(reader, 0))) + } + + case "Double precision FP": + if column.SpaceUsage == 8 { + result.Set(column.Name, math.Float64frombits( + ParseUint64(reader, 0))) + } + + case "DateTime": + if column.SpaceUsage == 8 { + switch column.Flags { + case 1: + // A more modern way of encoding + result.Set(column.Name, WinFileTime64(reader, 0)) + + case 0: + // Some hair brained time serialization method + // https://docs.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-coltyp + + value_int := ParseUint64(reader, 0) + days_since_1900 := math.Float64frombits(value_int) + + // In python time.mktime((1900,1,1,0,0,0,0,365,0)) + result.Set(column.Name, + time.Unix(int64(days_since_1900*24*60*60)+ + -2208988800, 0).UTC()) + + default: + // We have no idea + result.Set(column.Name, ParseUint64(reader, 0)) + } + } + + case "Long long", "Currency": + if column.SpaceUsage == 8 { + result.Set(column.Name, ParseUint64(tag.Reader, 0)) + } + + case "GUID": + if column.SpaceUsage == 16 { + result.Set(column.Name, + self.Header.Profile.GUID(tag.Reader, 0).AsString()) + } default: if Debug { @@ -306,6 +395,31 @@ func getSlice(buffer []byte, start, end uint64) []byte { return buffer[start:end] } +// working slice to reassemble data +type tagBuffer struct { + identifier uint32 + start, end uint64 + flags uint64 +} + +/* + Tagged values are used to store sparse values. + + The consist of an array of RecordTag, each RecordTag has an + Identifier and an offset to the start of its data. The length of the + data in each record is determine by the start of the next record. + + Example: + + 00000050 00 01 0c 40 a4 01 21 00 a5 01 23 00 01 6c 00 61 |...@..!...#..l.a| + 00000060 00 62 00 5c 00 64 00 63 00 2d 00 31 00 24 00 00 |.b.\.d.c.-.1.$..| + 00000070 00 3d 00 f9 00 |.=...| + + Slice is 0x50-0x75 00010c40a4012100a5012300016c00610062005c00640063002d003100240000003d00f900 + Consumed 0x15 bytes of TAGGED space from 0xc to 0x21 for tag 0x100 + Consumed 0x2 bytes of TAGGED space from 0x21 to 0x23 for tag 0x1a4 + Consumed 0x2 bytes of TAGGED space from 0x23 to 0x25 for tag 0x1a5 +*/ func ParseTaggedValues(ctx *ESEContext, buffer []byte) map[uint32][]byte { result := make(map[uint32][]byte) @@ -315,36 +429,40 @@ func ParseTaggedValues(ctx *ESEContext, buffer []byte) map[uint32][]byte { reader := &BufferReaderAt{buffer} first_record := ctx.Profile.RecordTag(reader, 0) - prev_record := first_record - - // Iterate over all tag headers - the headers go until the - // start of the first data segment - for offset := uint64(first_record.Size()); offset < first_record.DataOffset(); offset += uint64(first_record.Size()) { - record := ctx.Profile.RecordTag(reader, int64(offset)) - result[uint32(prev_record.Identifier())] = getSlice(buffer, prev_record.DataOffset()+ - prev_record.FlagSkip(), record.DataOffset()) + tags := []tagBuffer{} + // Tags go from 0 to the start of the first tag's data + for offset := int64(0); offset < int64(first_record.DataOffset()); offset += 4 { + record_tag := ctx.Profile.RecordTag(reader, offset) if Debug { - fmt.Printf("Consumed %#x bytes of TAGGED space from %#x for tag %#x\n", - record.DataOffset()-prev_record.DataOffset()-prev_record.FlagSkip(), - prev_record.DataOffset()+prev_record.FlagSkip(), - prev_record.Identifier()) + fmt.Printf("RecordTag %v\n", record_tag.DebugString()) } - - prev_record = record + tags = append(tags, tagBuffer{ + identifier: uint32(record_tag.Identifier()), + start: record_tag.DataOffset(), + flags: record_tag.Flags(), + }) } - // Last record goes to the end of the buffer. - result[uint32(prev_record.Identifier())] = getSlice(buffer, prev_record.DataOffset()+ - prev_record.FlagSkip(), uint64(len(buffer))) + // Now build a map from identifier to buffer. + for idx, tag := range tags { + // The last tag goes until the end of the buffer + end := uint64(len(buffer)) + start := tag.start + if idx < len(tags)-1 { + end = tags[idx+1].start + } - if Debug { - fmt.Printf("Consumed %#x bytes of TAGGED space from %#x for tag %#x\n", - uint64(len(buffer))-prev_record.DataOffset()-prev_record.FlagSkip(), - prev_record.DataOffset()+prev_record.FlagSkip(), - prev_record.Identifier()) - } + if tag.flags > 0 { + start += 1 + } + result[tag.identifier] = buffer[start:end] + if Debug { + fmt.Printf("Consumed %#x bytes of TAGGED space from %#x to %#x for tag %#x\n", + end-start, start, end, tag.identifier) + } + } return result } diff --git a/ese_gen.go b/ese_gen.go index cfe56f3..e788235 100644 --- a/ese_gen.go +++ b/ese_gen.go @@ -112,13 +112,14 @@ type ESEProfile struct { Off_PageHeader_Flags int64 Off_RecordTag_Identifier int64 Off_RecordTag_DataOffset int64 + Off_RecordTag_Flags int64 Off_Tag__ValueSize int64 Off_Tag__ValueOffset int64 } func NewESEProfile() *ESEProfile { // Specific offsets can be tweaked to cater for slight version mismatches. - self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,0,-2,0,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,4,6,8,0,1,2,3,4,5,4,12,0,0,0,0,0,8,16,20,24,28,32,34,36,0,2,0,2} + self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,0,-2,0,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,4,6,8,0,1,2,3,4,5,4,12,0,0,0,0,0,8,16,20,24,28,32,34,36,0,2,2,0,2} return self } @@ -963,6 +964,10 @@ func (self *PageHeader) Flags() *Flags { names := make(map[string]bool) + if value & 1 != 0 { + names["Root"] = true + } + if value & 2 != 0 { names["Leaf"] = true } @@ -987,10 +992,6 @@ func (self *PageHeader) Flags() *Flags { names["Long"] = true } - if value & 1 != 0 { - names["Root"] = true - } - return &Flags{Value: uint64(value), Names: names} } @@ -1025,10 +1026,16 @@ func (self *RecordTag) DataOffset() uint64 { value := ParseUint16(self.Reader, self.Profile.Off_RecordTag_DataOffset + self.Offset) return (uint64(value) & 0x1fff) >> 0x0 } + +func (self *RecordTag) Flags() uint64 { + value := ParseUint16(self.Reader, self.Profile.Off_RecordTag_Flags + self.Offset) + return (uint64(value) & 0xffff) >> 0xe +} func (self *RecordTag) DebugString() string { result := fmt.Sprintf("struct RecordTag @ %#x:\n", self.Offset) result += fmt.Sprintf(" Identifier: %#0x\n", self.Identifier()) result += fmt.Sprintf(" DataOffset: %#0x\n", self.DataOffset()) + result += fmt.Sprintf(" Flags: %#0x\n", self.Flags()) return result } diff --git a/ese_profile.json b/ese_profile.json index e9483d4..eec31f3 100644 --- a/ese_profile.json +++ b/ese_profile.json @@ -185,6 +185,11 @@ "target": "unsigned short", "start_bit": 0, "end_bit": 13 + }]], + "Flags": [2, ["BitField", { + "target": "unsigned short", + "start_bit": 14, + "end_bit": 16 }]] }], From c5d39d5aa33a37a819b60b5a7cd3fac271e5bd30 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Thu, 21 Oct 2021 19:07:06 +1000 Subject: [PATCH 25/40] Decode uint64 correctly from JSON (#11) --- .github/workflows/test.yaml | 20 ++++++++++++++++++++ ordereddict.go | 27 ++++++++++++++++++++++----- ordereddict_test.go | 24 +++++++++++++++++------- 3 files changed, 59 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/test.yaml diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml new file mode 100644 index 0000000..db0b9e6 --- /dev/null +++ b/.github/workflows/test.yaml @@ -0,0 +1,20 @@ +name: Test +on: [pull_request] +jobs: + build: + name: Windows Test + runs-on: ubuntu-18.04 + steps: + - name: Set up Go 1.17 + uses: actions/setup-go@v2 + with: + go-version: 1.17 + id: go + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + + - name: Test + shell: bash + if: always() + run: | + go test -v ./... diff --git a/ordereddict.go b/ordereddict.go index b92888d..ab42695 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "reflect" + "strconv" "strings" "sync" ) @@ -98,7 +99,7 @@ func (self *Dict) Update(key string, value interface{}) *Dict { if pres { self.store[key] = value } else { - self.Set(key, value) + self.set(key, value) } return self @@ -108,6 +109,10 @@ func (self *Dict) Set(key string, value interface{}) *Dict { self.Lock() defer self.Unlock() + return self.set(key, value) +} + +func (self *Dict) set(key string, value interface{}) *Dict { // O(n) but for our use case this is faster since Dicts are // typically small and we rarely overwrite a key. _, pres := self.store[key] @@ -262,7 +267,9 @@ func (self *Dict) GoString() string { return self.String() } -// this implements type json.Unmarshaler interface, so can be called in json.Unmarshal(data, om) +// this implements type json.Unmarshaler interface, so can be called +// in json.Unmarshal(data, om). We preserve key order when +// unmarshaling from JSON. func (self *Dict) UnmarshalJSON(data []byte) error { self.Lock() defer self.Unlock() @@ -391,12 +398,21 @@ func handledelim(token json.Token, dec *json.Decoder) (res interface{}, err erro } case json.Number: - value, err := t.Int64() + value_str := t.String() + + // Try to parse as Uint + value_uint, err := strconv.ParseUint(value_str, 10, 64) if err == nil { - return value, nil + return value_uint, nil + } + + value_int, err := strconv.ParseInt(value_str, 10, 64) + if err == nil { + return value_int, nil } - float, err := t.Float64() + // Failing this, try a float + float, err := strconv.ParseFloat(value_str, 64) if err == nil { return float, nil } @@ -406,6 +422,7 @@ func handledelim(token json.Token, dec *json.Decoder) (res interface{}, err erro return token, nil } +// Preserve key order when marshalling to JSON. func (self *Dict) MarshalJSON() ([]byte, error) { self.Lock() defer self.Unlock() diff --git a/ordereddict_test.go b/ordereddict_test.go index fbaccf7..6785bca 100644 --- a/ordereddict_test.go +++ b/ordereddict_test.go @@ -43,20 +43,30 @@ var ( // Check that serialization decodes to the object. dictUnserializationTest = []dictSerializationTest{ // Preserve order of keys on deserialization. - {NewDict().Set("A", int64(1)).Set("B", int64(2)), `{"A":1,"B":2}`}, - {NewDict().Set("B", int64(1)).Set("A", int64(2)), `{"B":1,"A":2}`}, + {NewDict().Set("A", uint64(1)).Set("B", uint64(2)), `{"A":1,"B":2}`}, + {NewDict().Set("B", uint64(1)).Set("A", uint64(2)), `{"B":1,"A":2}`}, // Handle arrays, ints floats and bools - {NewDict().Set("B", int64(1)).Set("A", int64(2)), `{"B":1,"A":2}`}, - {NewDict().Set("B", float64(1)).Set("A", int64(2)), `{"B":1.0,"A":2}`}, - {NewDict().Set("B", []interface{}{int64(1)}).Set("A", int64(2)), `{"B":[1],"A":2}`}, - {NewDict().Set("B", true).Set("A", int64(2)), `{"B":true,"A":2}`}, - {NewDict().Set("B", nil).Set("A", int64(2)), `{"B":null,"A":2}`}, + {NewDict().Set("B", uint64(1)).Set("A", uint64(2)), `{"B":1,"A":2}`}, + {NewDict().Set("B", float64(1)).Set("A", uint64(2)), `{"B":1.0,"A":2}`}, + {NewDict().Set("B", []interface{}{uint64(1)}).Set("A", uint64(2)), `{"B":[1],"A":2}`}, + {NewDict().Set("B", true).Set("A", uint64(2)), `{"B":true,"A":2}`}, + {NewDict().Set("B", nil).Set("A", uint64(2)), `{"B":null,"A":2}`}, // Embedded dicts decode into ordered dicts. {NewDict(). Set("B", NewDict().Set("Zoo", "X").Set("Baz", "Y")). Set("A", "Z"), `{"B":{"Zoo":"X","Baz":"Y"},"A":"Z"}`}, + + // Make sure we properly preserve uint64 (overflows int64) + {NewDict(). + Set("Uint64", uint64(9223372036854775808)), + `{"Uint64": 9223372036854775808}`}, + + // We prefer uint64 but int64 is needed for negative numbers + {NewDict(). + Set("Int64", int64(-500)), + `{"Int64": -500}`}, } ) From bd222af275405178b9de67033ec02fbc348d5123 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Wed, 3 Nov 2021 02:02:26 +1000 Subject: [PATCH 26/40] Fixed bug in parsing ESE Datetimes (#13) Turns out the documentation is misleading - the time represents the number of days since Dec 30 1899 not Jan 1 1900 --- catalog.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/catalog.go b/catalog.go index 05a1b47..d50929b 100644 --- a/catalog.go +++ b/catalog.go @@ -175,9 +175,31 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { days_since_1900 := math.Float64frombits(value_int) // In python time.mktime((1900,1,1,0,0,0,0,365,0)) + + // From https://docs.microsoft.com/en-us/windows/win32/api/oleauto/nf-oleauto-varianttimetosystemtime + // A variant time is stored as an 8-byte real + // value (double), representing a date between + // January 1, 100 and December 31, 9999, + // inclusive. The value 2.0 represents January + // 1, 1900; 3.0 represents January 2, 1900, + // and so on. Adding 1 to the value increments + // the date by a day. The fractional part of + // the value represents the time of + // day. Therefore, 2.5 represents noon on + // January 1, 1900; 3.25 represents 6:00 + // A.M. on January 2, 1900, and so + // on. Negative numbers represent the dates + // prior to December 30, 1899. result.Set(column.Name, - time.Unix(int64(days_since_1900*24*60*60)+ - -2208988800, 0).UTC()) + time.Unix(int64( + days_since_1900*24*60*60)+ + + // Number of Sec between 1900 and 1970 + -2208988800- + + // Jan 1 1900 is actually value of 2 + // days so correct for it here. + 2*24*60*60, 0).UTC()) default: // We have no idea From c8e40f18702c4536235902304e6ad5cfeac5732f Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Thu, 9 Dec 2021 14:17:27 +1000 Subject: [PATCH 27/40] Added GetBool() convenience function (#12) --- ordereddict.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ordereddict.go b/ordereddict.go index ab42695..c77faff 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -168,6 +168,17 @@ func (self *Dict) GetString(key string) (string, bool) { return "", false } +func (self *Dict) GetBool(key string) (bool, bool) { + v, pres := self.Get(key) + if pres { + v_bool, ok := v.(bool) + if ok { + return v_bool, true + } + } + return false, false +} + func to_string(x interface{}) (string, bool) { switch t := x.(type) { case string: From 572009c595d061f013038260c002380d05bf12f3 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Thu, 23 Dec 2021 18:25:14 +1000 Subject: [PATCH 28/40] Support ordered yaml encoding (#13) --- go.mod | 1 + go.sum | 2 ++ ordereddict.go | 17 +++++++++++++++++ 3 files changed, 20 insertions(+) diff --git a/go.mod b/go.mod index b01afb5..9346c59 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/Velocidex/ordereddict go 1.13 require ( + github.com/Velocidex/yaml/v2 v2.2.5 // indirect github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 github.com/stretchr/testify v1.4.0 ) diff --git a/go.sum b/go.sum index 8b718bb..224e6a8 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Velocidex/yaml/v2 v2.2.5 h1:8XZwR8tmm5UWAXotI3fL17s9fjKEMqZ293fgqUiMhBw= +github.com/Velocidex/yaml/v2 v2.2.5/go.mod h1:VBjrsTMc/b1h0ankOOnJPYoCbJNwhpGYpnDgICEs2mk= github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E= github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= diff --git a/ordereddict.go b/ordereddict.go index c77faff..5661950 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -9,6 +9,8 @@ import ( "strconv" "strings" "sync" + + "github.com/Velocidex/yaml/v2" ) // A concerete implementation of a row - similar to Python's @@ -470,3 +472,18 @@ func (self *Dict) MarshalJSON() ([]byte, error) { result = result + "}" return []byte(result), nil } + +func (self *Dict) MarshalYAML() (interface{}, error) { + self.Lock() + defer self.Unlock() + + result := yaml.MapSlice{} + for _, k := range self.keys { + v := self.store[k] + result = append(result, yaml.MapItem{ + Key: k, Value: v, + }) + } + + return result, nil +} From 3dbe58412844aae1c0a4d1af516fe39d688b179f Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Fri, 7 Jan 2022 17:50:49 +1000 Subject: [PATCH 29/40] Sync deps (#14) --- go.mod | 8 +++++--- go.sum | 21 ++++++++++++--------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 9346c59..083ab7f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,9 @@ module github.com/Velocidex/ordereddict go 1.13 require ( - github.com/Velocidex/yaml/v2 v2.2.5 // indirect - github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 - github.com/stretchr/testify v1.4.0 + github.com/Velocidex/yaml/v2 v2.2.8 + github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/stretchr/testify v1.7.0 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/go.sum b/go.sum index 224e6a8..53373cf 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,17 @@ -github.com/Velocidex/yaml/v2 v2.2.5 h1:8XZwR8tmm5UWAXotI3fL17s9fjKEMqZ293fgqUiMhBw= -github.com/Velocidex/yaml/v2 v2.2.5/go.mod h1:VBjrsTMc/b1h0ankOOnJPYoCbJNwhpGYpnDgICEs2mk= -github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1 h1:GDQdwm/gAcJcLAKQQZGOJ4knlw+7rfEQQcmwTbt4p5E= -github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/Velocidex/yaml/v2 v2.2.8 h1:GUrSy4SBJ6RjGt43k6MeBKtw2z/27gh4A3hfFmFY3No= +github.com/Velocidex/yaml/v2 v2.2.8/go.mod h1:PlXIg/Pxmoja48C1vMHo7C5pauAZvLq/UEPOQ3DsjS4= +github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= +github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 79032cf99b1d5fd29b005b3caa1208ec5a7f2393 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Mon, 11 Apr 2022 20:34:15 +1000 Subject: [PATCH 30/40] Optimize Dict serialization. (#15) --- go.mod | 1 + go.sum | 2 ++ ordereddict.go | 20 ++++++++++++-------- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index 083ab7f..8b8d3d7 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/Velocidex/ordereddict go 1.13 require ( + github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a // indirect github.com/Velocidex/yaml/v2 v2.2.8 github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 github.com/davecgh/go-spew v1.1.1 // indirect diff --git a/go.sum b/go.sum index 53373cf..370e1e7 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a h1:AeXPUzhU0yhID/v5JJEIkjaE85ASe+Vh4Kuv1RSLL+4= +github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a/go.mod h1:ukJBuruT9b24pdgZwWDvOaCYHeS03B7oQPCUWh25bwM= github.com/Velocidex/yaml/v2 v2.2.8 h1:GUrSy4SBJ6RjGt43k6MeBKtw2z/27gh4A3hfFmFY3No= github.com/Velocidex/yaml/v2 v2.2.8/go.mod h1:PlXIg/Pxmoja48C1vMHo7C5pauAZvLq/UEPOQ3DsjS4= github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= diff --git a/ordereddict.go b/ordereddict.go index 5661950..c202f43 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -2,7 +2,6 @@ package ordereddict import ( "bytes" - "encoding/json" "fmt" "io" "reflect" @@ -10,6 +9,7 @@ import ( "strings" "sync" + "github.com/Velocidex/json" "github.com/Velocidex/yaml/v2" ) @@ -440,7 +440,8 @@ func (self *Dict) MarshalJSON() ([]byte, error) { self.Lock() defer self.Unlock() - result := "{" + buf := &bytes.Buffer{} + buf.Write([]byte("{")) for _, k := range self.keys { // add key @@ -458,19 +459,22 @@ func (self *Dict) MarshalJSON() ([]byte, error) { continue } - result += string(kEscaped) + ":" + buf.Write(kEscaped) + buf.Write([]byte(":")) + vBytes, err := json.Marshal(v) if err == nil { - result += string(vBytes) + "," + buf.Write(vBytes) + buf.Write([]byte(",")) } else { - result += "null," + buf.Write([]byte("null,")) } } if len(self.keys) > 0 { - result = result[0 : len(result)-1] + buf.Truncate(buf.Len() - 1) } - result = result + "}" - return []byte(result), nil + buf.Write([]byte("}")) + return buf.Bytes(), nil } func (self *Dict) MarshalYAML() (interface{}, error) { From da46091cd216024cb9f6d0e2fb2cacb0faf94826 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Fri, 29 Apr 2022 01:34:15 +1000 Subject: [PATCH 31/40] Fix crash when unmarshaling zero value (#16) --- ordereddict.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ordereddict.go b/ordereddict.go index c202f43..63c9c79 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -340,6 +340,10 @@ func (self *Dict) parseobject(dec *json.Decoder) (err error) { return err } self.keys = append(self.keys, key) + if self.store == nil { + self.store = make(map[string]interface{}) + } + self.store[key] = value if self.case_map != nil { self.case_map[strings.ToLower(key)] = key From 6a7cb85851cdeda3016768b4e6d765aa4ab0a6e7 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Thu, 10 Nov 2022 23:07:14 +1000 Subject: [PATCH 32/40] Decode time objects from JSON (#17) --- Makefile | 2 ++ go.mod | 8 +++----- go.sum | 16 ++++++++++------ ordereddict.go | 13 +++++++++++++ ordereddict_test.go | 22 ++++++++++++++++++++++ 5 files changed, 50 insertions(+), 11 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8131898 --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +all: + go test -v . diff --git a/go.mod b/go.mod index 8b8d3d7..efce6ab 100644 --- a/go.mod +++ b/go.mod @@ -3,10 +3,8 @@ module github.com/Velocidex/ordereddict go 1.13 require ( - github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a // indirect + github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a github.com/Velocidex/yaml/v2 v2.2.8 - github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/stretchr/testify v1.7.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + github.com/alecthomas/repr v0.1.1 + github.com/stretchr/testify v1.8.1 ) diff --git a/go.sum b/go.sum index 370e1e7..a44f287 100644 --- a/go.sum +++ b/go.sum @@ -2,18 +2,22 @@ github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a h1:AeXPUzhU0yhID/v5 github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a/go.mod h1:ukJBuruT9b24pdgZwWDvOaCYHeS03B7oQPCUWh25bwM= github.com/Velocidex/yaml/v2 v2.2.8 h1:GUrSy4SBJ6RjGt43k6MeBKtw2z/27gh4A3hfFmFY3No= github.com/Velocidex/yaml/v2 v2.2.8/go.mod h1:PlXIg/Pxmoja48C1vMHo7C5pauAZvLq/UEPOQ3DsjS4= -github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= -github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= +github.com/alecthomas/repr v0.1.1 h1:87P60cSmareLAxMc4Hro0r2RBY4ROm0dYwkJNpS4pPs= +github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/ordereddict.go b/ordereddict.go index 63c9c79..0a1aaff 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" "sync" + "time" "github.com/Velocidex/json" "github.com/Velocidex/yaml/v2" @@ -414,6 +415,18 @@ func handledelim(token json.Token, dec *json.Decoder) (res interface{}, err erro return nil, fmt.Errorf("Unexpected delimiter: %q", t) } + case string: + // does it look like a timestamp in RFC3339 + if len(t) >= 20 && t[10] == 'T' { + // Attempt to convert it from timestamp. + parsed, err := time.Parse(time.RFC3339, t) + if err == nil { + return parsed, nil + } + } + + return t, nil + case json.Number: value_str := t.String() diff --git a/ordereddict_test.go b/ordereddict_test.go index 6785bca..e2359ee 100644 --- a/ordereddict_test.go +++ b/ordereddict_test.go @@ -97,6 +97,28 @@ func TestDictDeserialization(t *testing.T) { } } +func TestTimestampDeserialization(t *testing.T) { + serialized := `{"Time":"2022-08-21T03:08:45.076884Z", "Time2":"2022-08-21T03:08:45.076884+07:00"}` + value := NewDict() + err := json.Unmarshal([]byte(serialized), value) + assert.NoError(t, err) + + t1_any, pres := value.Get("Time") + assert.True(t, pres) + + t1, ok := t1_any.(time.Time) + assert.True(t, ok) + assert.Equal(t, int64(1661051325), t1.Unix()) + + t2_any, pres := value.Get("Time2") + assert.True(t, pres) + + t2, ok := t2_any.(time.Time) + assert.True(t, ok) + assert.Equal(t, int64(1661026125), t2.Unix()) + +} + func TestOrder(t *testing.T) { test := NewDict(). Set("A", 1). From c3dbe8034342672ff789f37880cc9bfc7f10875c Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Mon, 21 Aug 2023 13:02:56 +1000 Subject: [PATCH 33/40] Support 7 bit compression in large text fields (#17) These are used in WebCacheV01.dat and Windows.edb commonly --- catalog.go | 25 +++++++---- compression.go | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ debug.go | 4 ++ 3 files changed, 133 insertions(+), 8 deletions(-) create mode 100644 compression.go diff --git a/catalog.go b/catalog.go index d50929b..c2e6fb5 100644 --- a/catalog.go +++ b/catalog.go @@ -218,6 +218,14 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { self.Header.Profile.GUID(tag.Reader, offset).AsString()) } + case "Binary": + if column.SpaceUsage < 1024 { + data := make([]byte, column.SpaceUsage) + n, err := tag.Reader.ReadAt(data, offset) + if err == nil { + result.Set(column.Name, data[:n]) + } + } default: fmt.Printf("Can not handle Column %v fixed data %v\n", column.Name, column) @@ -247,15 +255,15 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { } else { switch column.Type { case "Binary": - result.Set(column.Name, ParseString(tag.Reader, - variableSizeOffset+variableDataBytesProcessed, - itemLen-prevItemLen)) + result.Set(column.Name, hex.EncodeToString([]byte( + ParseString(tag.Reader, + variableSizeOffset+variableDataBytesProcessed, + itemLen-prevItemLen)))) case "Text": - result.Set(column.Name, ParseString( - tag.Reader, + result.Set(column.Name, ParseText(tag.Reader, variableSizeOffset+variableDataBytesProcessed, - itemLen-prevItemLen)) + itemLen-prevItemLen, column.Flags)) default: fmt.Printf("Can not handle Column %v variable data %v\n", @@ -297,8 +305,9 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { result.Set(column.Name, hex.EncodeToString(buf)) case "Long Text": - result.Set(column.Name, ParseTerminatedUTF16String( - reader, 0)) + // Flags can be given as the first char or in the + // column definition. + result.Set(column.Name, ParseLongText(buf, column.Flags)) case "Boolean": if column.SpaceUsage == 1 { diff --git a/compression.go b/compression.go new file mode 100644 index 0000000..e1c29f5 --- /dev/null +++ b/compression.go @@ -0,0 +1,112 @@ +package parser + +import ( + "encoding/binary" + "fmt" + "io" + "strings" +) + +const ( + // Flags on the record header + COMPRESSION = 0x02 + INLINE_STRING = 0x01 + INLINE_STRING_2 = 0x08 + LZMA_COMPRESSION = 0x18 // Not supported +) + +func Decompress7BitCompression(buf []byte) string { + result := make([]byte, 0, (len(buf)+5)*8/7) + + value_16bit := uint16(0) + bit_index := 0 + + for i := 1; i < len(buf); i++ { + slice := buf[i : i+1] + if i+1 < len(buf) { + slice = append(slice, buf[i+1]) + } + + for len(slice) < 2 { + slice = append(slice, 0) + } + + value_16bit |= binary.LittleEndian.Uint16(slice) << bit_index + result = append(result, byte(value_16bit&0x7f)) + + value_16bit >>= 7 + bit_index++ + + if bit_index == 7 { + result = append(result, byte(value_16bit&0x7f)) + value_16bit >>= 7 + bit_index = 0 + } + } + + return strings.TrimSuffix(string(result), "\x00") +} + +func ParseLongText(buf []byte, flag uint32) string { + if len(buf) < 2 { + return "" + } + + // fmt.Printf("Column Flags %v\n", flag) + leading_byte := buf[0] + if leading_byte != 0 && leading_byte != 1 && leading_byte != 8 && + leading_byte != 3 && leading_byte != 0x18 { + return strings.TrimSuffix( + UTF16BytesToUTF8(buf, binary.LittleEndian), "\x00") + + } + // fmt.Printf("Inline Flags %v\n", flag) + + // Lzxpress compression - not supported right now. + if leading_byte == 0x18 { + fmt.Printf("LZXPRESS compression not supported currently\n") + return string(buf) + } + + // The following is either 7 bit compressed or utf16 encoded. Its + // hard to figure out which it is though because there is no + // consistency in the flags. We do our best to guess!! + var result string + if len(buf) >= 3 && buf[2] == 0 { + // Probably UTF16 encoded + result = strings.TrimSuffix( + UTF16BytesToUTF8(buf[1:], binary.LittleEndian), "\x00") + + } else { + // Probably 7bit compressed + result = Decompress7BitCompression(buf[1:]) + } + + //fmt.Printf("returned %v\n", result) + return result +} + +func ParseText(reader io.ReaderAt, offset int64, len int64, flags uint32) string { + if len < 0 { + return "" + + } + if len > 1024*10 { + len = 1024 * 10 + } + + data := make([]byte, len) + n, err := reader.ReadAt(data, offset) + if err != nil { + return "" + } + data = data[:n] + + var str string + if flags == 1 { + str = string(data[:n]) + } else { + str = UTF16BytesToUTF8(data, binary.LittleEndian) + } + return strings.TrimSuffix(str, "\x00") +} diff --git a/debug.go b/debug.go index e37ad9e..bbda069 100644 --- a/debug.go +++ b/debug.go @@ -7,3 +7,7 @@ var ( // Debugging during walk DebugWalk = false ) + +func DlvDebug() { + +} From 93923445369e100f023246a73e82ae7ac91f162b Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Mon, 21 Aug 2023 13:19:27 +1000 Subject: [PATCH 34/40] Build with ubuntu (#19) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c89ffb2..25cc1e9 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ There are three types of columns: - Fixed size (e.g. integers) have a known size. - Variable size (e.g. Strings) have a variable size. - Tagged data - these columns are often null and therefore may not be - present. The database stored these with their column ID as a map. + present. The database stores these with their column ID as a map. Therefore within the tag for each column, there are three distinct storage areas. You can see how each record is parsed using the --debug flag: From e26c2c8e5c31708c6fd724ed4614971b6f7d8bfe Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Mon, 21 Aug 2023 21:44:11 +1000 Subject: [PATCH 35/40] Null Terminate strings (#20) --- compression.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/compression.go b/compression.go index e1c29f5..0606c7d 100644 --- a/compression.go +++ b/compression.go @@ -44,7 +44,7 @@ func Decompress7BitCompression(buf []byte) string { } } - return strings.TrimSuffix(string(result), "\x00") + return strings.Split(string(result), "\x00")[0] } func ParseLongText(buf []byte, flag uint32) string { @@ -56,8 +56,8 @@ func ParseLongText(buf []byte, flag uint32) string { leading_byte := buf[0] if leading_byte != 0 && leading_byte != 1 && leading_byte != 8 && leading_byte != 3 && leading_byte != 0x18 { - return strings.TrimSuffix( - UTF16BytesToUTF8(buf, binary.LittleEndian), "\x00") + return strings.Split( + UTF16BytesToUTF8(buf, binary.LittleEndian), "\x00")[0] } // fmt.Printf("Inline Flags %v\n", flag) @@ -65,7 +65,7 @@ func ParseLongText(buf []byte, flag uint32) string { // Lzxpress compression - not supported right now. if leading_byte == 0x18 { fmt.Printf("LZXPRESS compression not supported currently\n") - return string(buf) + return strings.Split(string(buf), "\x00")[0] } // The following is either 7 bit compressed or utf16 encoded. Its @@ -74,8 +74,7 @@ func ParseLongText(buf []byte, flag uint32) string { var result string if len(buf) >= 3 && buf[2] == 0 { // Probably UTF16 encoded - result = strings.TrimSuffix( - UTF16BytesToUTF8(buf[1:], binary.LittleEndian), "\x00") + result = UTF16BytesToUTF8(buf[1:], binary.LittleEndian) } else { // Probably 7bit compressed @@ -83,7 +82,7 @@ func ParseLongText(buf []byte, flag uint32) string { } //fmt.Printf("returned %v\n", result) - return result + return strings.Split(result, "\x00")[0] } func ParseText(reader io.ReaderAt, offset int64, len int64, flags uint32) string { @@ -108,5 +107,5 @@ func ParseText(reader io.ReaderAt, offset int64, len int64, flags uint32) string } else { str = UTF16BytesToUTF8(data, binary.LittleEndian) } - return strings.TrimSuffix(str, "\x00") + return strings.Split(str, "\x00")[0] } From 2aa49cc5d11d51b06db3d0234aea849a5c8f5e5d Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Sun, 10 Sep 2023 03:41:57 +1000 Subject: [PATCH 36/40] Implement custom yaml unmarshal (#18) --- .github/workflows/test.yaml | 10 +++++----- ordereddict.go | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index db0b9e6..59e0dcd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -3,15 +3,15 @@ on: [pull_request] jobs: build: name: Windows Test - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - - name: Set up Go 1.17 - uses: actions/setup-go@v2 + - name: Set up Go 1.20 + uses: actions/setup-go@v3 with: - go-version: 1.17 + go-version: '^1.20' id: go - name: Check out code into the Go module directory - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Test shell: bash diff --git a/ordereddict.go b/ordereddict.go index 0a1aaff..a97df08 100644 --- a/ordereddict.go +++ b/ordereddict.go @@ -125,6 +125,10 @@ func (self *Dict) set(key string, value interface{}) *Dict { self.keys = append(self.keys, key) } + if self.store == nil { + self.store = make(map[string]interface{}) + } + self.store[key] = value if self.case_map != nil { @@ -281,6 +285,22 @@ func (self *Dict) GoString() string { return self.String() } +func (self *Dict) UnmarshalYAML(unmarshal func(interface{}) error) error { + m := yaml.MapSlice{} + err := unmarshal(&m) + if err != nil { + return err + } + + for _, item := range m { + key, ok := item.Key.(string) + if ok { + self.Set(key, item.Value) + } + } + return nil +} + // this implements type json.Unmarshaler interface, so can be called // in json.Unmarshal(data, om). We preserve key order when // unmarshaling from JSON. From 744a60520467255c4e384ce0982e0ae14365e43a Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Wed, 7 Feb 2024 04:09:15 +1000 Subject: [PATCH 37/40] Implemented long value support (#22) I discovered that the source code for ESE was published by Microsoft and so this work is based around reading the original source code. Added more documentation to the file format. --- catalog.go | 254 +++++++++++++++++++++++++++++++++---------- context.go | 9 +- conversion.spec.yaml | 14 ++- debug.go | 12 ++ ese_gen.go | 215 ++++++++++++++++++++++++++++-------- ese_profile.json | 41 ++++++- long_values.go | 130 ++++++++++++++++++++++ pages.go | 114 ++++++++++++++----- reader.go | 28 +++++ utils.go | 4 + 10 files changed, 678 insertions(+), 143 deletions(-) create mode 100644 long_values.go diff --git a/catalog.go b/catalog.go index c2e6fb5..597e35f 100644 --- a/catalog.go +++ b/catalog.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "math" + "sort" "time" "github.com/Velocidex/ordereddict" @@ -15,6 +16,9 @@ import ( const ( CATALOG_PAGE_NUMBER = 4 + + // https://github.com/microsoft/Extensible-Storage-Engine/blob/933dc839b5a97b9a5b3e04824bdd456daf75a57d/dev/ese/src/inc/node.hxx#L226 + fNDCompressed = 4 << 13 ) // Store a simple struct of column spec for speed. @@ -34,7 +38,7 @@ type Table struct { Name string Columns []*ColumnSpec Indexes *ordereddict.Dict - LongValues *ordereddict.Dict + LongValueLookup LongValueLookup } // The tag contains a single row. @@ -77,22 +81,23 @@ type Table struct { // Then the tagged values are consumed // Column RDomain Identifier 256 Type Long Text -func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { - tag := NewESENT_LEAF_ENTRY(self.ctx, value) - +func (self *Table) tagToRecord(value *Value, header *PageHeader) *ordereddict.Dict { if Debug { fmt.Printf("Processing row in Tag @ %d %#x (%#x)", - value.Tag.Offset, value.Tag.ValueOffset(self.ctx), + value.Tag.Offset, + value.Tag.ValueOffsetInPage(self.ctx, header), value.Tag.ValueSize(self.ctx)) - spew.Dump(value.Buffer) - tag.Dump() + spew.Dump(value.GetBuffer()) } result := ordereddict.NewDict() var taggedItems map[uint32][]byte - dd_header := self.ctx.Profile.ESENT_DATA_DEFINITION_HEADER(tag.Reader, tag.EntryData()) + reader := value.Reader() + + tag := NewESENT_LEAF_ENTRY(self.ctx, value) + dd_header := self.ctx.Profile.ESENT_DATA_DEFINITION_HEADER(reader, tag.EntryData()) // Start to parse immediately after the dd_header offset := dd_header.Offset + int64(dd_header.Size()) @@ -120,44 +125,44 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { switch column.Type { case "Boolean": if column.SpaceUsage == 1 { - result.Set(column.Name, ParseUint8(tag.Reader, offset) > 0) + result.Set(column.Name, ParseUint8(reader, offset) > 0) } case "Signed byte": if column.SpaceUsage == 1 { - result.Set(column.Name, ParseUint8(tag.Reader, offset)) + result.Set(column.Name, ParseUint8(reader, offset)) } case "Signed short": if column.SpaceUsage == 2 { - result.Set(column.Name, ParseInt16(tag.Reader, offset)) + result.Set(column.Name, ParseInt16(reader, offset)) } case "Unsigned short": if column.SpaceUsage == 2 { - result.Set(column.Name, ParseUint16(tag.Reader, offset)) + result.Set(column.Name, ParseUint16(reader, offset)) } case "Signed long": if column.SpaceUsage == 4 { - result.Set(column.Name, ParseInt32(tag.Reader, offset)) + result.Set(column.Name, ParseInt32(reader, offset)) } case "Unsigned long": if column.SpaceUsage == 4 { - result.Set(column.Name, ParseUint32(tag.Reader, offset)) + result.Set(column.Name, ParseUint32(reader, offset)) } case "Single precision FP": if column.SpaceUsage == 4 { result.Set(column.Name, math.Float32frombits( - ParseUint32(tag.Reader, offset))) + ParseUint32(reader, offset))) } case "Double precision FP": if column.SpaceUsage == 8 { result.Set(column.Name, math.Float64frombits( - ParseUint64(tag.Reader, offset))) + ParseUint64(reader, offset))) } case "DateTime": @@ -165,13 +170,13 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { switch column.Flags { case 1: // A more modern way of encoding - result.Set(column.Name, WinFileTime64(tag.Reader, offset)) + result.Set(column.Name, WinFileTime64(reader, offset)) case 0: // Some hair brained time serialization method // https://docs.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-coltyp - value_int := ParseUint64(tag.Reader, offset) + value_int := ParseUint64(reader, offset) days_since_1900 := math.Float64frombits(value_int) // In python time.mktime((1900,1,1,0,0,0,0,365,0)) @@ -203,25 +208,38 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { default: // We have no idea - result.Set(column.Name, ParseUint64(tag.Reader, offset)) + result.Set(column.Name, ParseUint64(reader, offset)) + } + } + + case "Long Text", "Text": + if column.SpaceUsage < 2000 { + data := make([]byte, column.SpaceUsage) + n, err := reader.ReadAt(data, offset) + + if err == nil { + + // Flags can be given as the first char or in the + // column definition. + result.Set(column.Name, ParseLongText(data[:n], column.Flags)) } } case "Long long", "Currency": if column.SpaceUsage == 8 { - result.Set(column.Name, ParseUint64(tag.Reader, offset)) + result.Set(column.Name, ParseUint64(reader, offset)) } case "GUID": if column.SpaceUsage == 16 { result.Set(column.Name, - self.Header.Profile.GUID(tag.Reader, offset).AsString()) + self.Header.Profile.GUID(reader, offset).AsString()) } case "Binary": if column.SpaceUsage < 1024 { data := make([]byte, column.SpaceUsage) - n, err := tag.Reader.ReadAt(data, offset) + n, err := reader.ReadAt(data, offset) if err == nil { result.Set(column.Name, data[:n]) } @@ -245,7 +263,7 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { // Variable data type index := int64(column.Identifier) - 127 - 1 - itemLen := int64(ParseUint16(tag.Reader, variableSizeOffset+index*2)) + itemLen := int64(ParseUint16(reader, variableSizeOffset+index*2)) if itemLen&0x8000 > 0 { // Empty Item @@ -256,12 +274,12 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { switch column.Type { case "Binary": result.Set(column.Name, hex.EncodeToString([]byte( - ParseString(tag.Reader, + ParseString(reader, variableSizeOffset+variableDataBytesProcessed, itemLen-prevItemLen)))) case "Text": - result.Set(column.Name, ParseText(tag.Reader, + result.Set(column.Name, ParseText(reader, variableSizeOffset+variableDataBytesProcessed, itemLen-prevItemLen, column.Flags)) @@ -285,26 +303,47 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { if Debug { fmt.Printf("Slice is %#x-%#x %x\n", variableDataBytesProcessed+variableSizeOffset, - len(value.Buffer), getSlice(value.Buffer, - uint64(variableDataBytesProcessed+ - variableSizeOffset), - uint64(len(value.Buffer)+1))) + value.BufferSize, + getValueSlice(value, uint64(variableDataBytesProcessed+ + variableSizeOffset), uint64(value.BufferSize))) } taggedItems = ParseTaggedValues( - self.ctx, getSlice(value.Buffer, - uint64(variableDataBytesProcessed+ - variableSizeOffset), - uint64(len(value.Buffer)+1))) + self.ctx, getValueSlice(value, + uint64(variableDataBytesProcessed+variableSizeOffset), + uint64(value.BufferSize))) } buf, pres := taggedItems[column.Identifier] if pres { reader := &BufferReaderAt{buf} switch column.Type { - case "Binary", "Long Binary": + case "Binary": + result.Set(column.Name, hex.EncodeToString(buf)) + + case "Long Binary": + // If the buf is key size (4 or 8 bytes) then we + // can look it up in the LV cache. Otherwise it is + // stored literally. + if len(buf) == 4 || len(buf) == 8 { + data, pres := self.LongValueLookup.GetLid(buf) + if pres { + buf = data + } + } + result.Set(column.Name, hex.EncodeToString(buf)) case "Long Text": + // If the buf is key size (4 or 8 bytes) then we + // can look it up in the LV cache. Otherwise it is + // stored literally. + if len(buf) == 4 || len(buf) == 8 { + data, pres := self.LongValueLookup.GetLid(buf) + if pres { + buf = data + } + } + // Flags can be given as the first char or in the // column definition. result.Set(column.Name, ParseLongText(buf, column.Flags)) @@ -378,13 +417,13 @@ func (self *Table) tagToRecord(value *Value) *ordereddict.Dict { case "Long long", "Currency": if column.SpaceUsage == 8 { - result.Set(column.Name, ParseUint64(tag.Reader, 0)) + result.Set(column.Name, ParseUint64(reader, 0)) } case "GUID": if column.SpaceUsage == 16 { result.Set(column.Name, - self.Header.Profile.GUID(tag.Reader, 0).AsString()) + self.Header.Profile.GUID(reader, 0).AsString()) } default: @@ -404,39 +443,33 @@ func (self *RecordTag) FlagSkip() uint64 { return 1 } -func getSlice(buffer []byte, start, end uint64) []byte { +func getValueSlice(value *Value, start, end uint64) []byte { if end < start { return nil } - length := uint64(len(buffer)) - - if start < 0 { - start = 0 - } - - if start > length { - start = length + length := end - start + if length > 1*1024*1024 { + return nil } - if end > length { - end = length - } + buffer := make([]byte, length) + value.reader.ReadAt(buffer, value.BufferOffset+int64(start)) - return buffer[start:end] + return buffer } // working slice to reassemble data type tagBuffer struct { - identifier uint32 - start, end uint64 - flags uint64 + identifier uint32 + start, length uint64 + flags uint64 } /* Tagged values are used to store sparse values. - The consist of an array of RecordTag, each RecordTag has an + They consist of an array of RecordTag, each RecordTag has an Identifier and an offset to the start of its data. The length of the data in each record is determine by the start of the next record. @@ -488,6 +521,18 @@ func ParseTaggedValues(ctx *ESEContext, buffer []byte) map[uint32][]byte { start += 1 } + if start > uint64(len(buffer)) { + start = uint64(len(buffer)) + } + + if end > uint64(len(buffer)) { + end = uint64(len(buffer)) + } + + if end < start { + end = start + } + result[tag.identifier] = buffer[start:end] if Debug { fmt.Printf("Consumed %#x bytes of TAGGED space from %#x to %#x for tag %#x\n", @@ -511,7 +556,11 @@ func (self *Catalog) DumpTable(name string, cb func(row *ordereddict.Dict) error func(header *PageHeader, id int64, value *Value) error { // Each tag stores a single row - all the // columns in the row are encoded in this tag. - return cb(table.tagToRecord(value)) + row := table.tagToRecord(value, header) + if len(row.Keys()) == 0 { + return nil + } + return cb(row) }) if err != nil { return err @@ -544,6 +593,8 @@ func parseItemName(dd_header *ESENT_DATA_DEFINITION_HEADER) string { 2*numEntries, int64(itemLen)) } +// Walking over each LINE in the catalog tree, we parse the data +// definitions. func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) error { leaf_entry := NewESENT_LEAF_ENTRY(self.ctx, value) dd_header := self.ctx.Profile.ESENT_DATA_DEFINITION_HEADER( @@ -563,7 +614,8 @@ func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) error Name: itemName, FatherDataPageNumber: catalog.Table().FatherDataPageNumber(), Indexes: ordereddict.NewDict(), - LongValues: ordereddict.NewDict()} + LongValueLookup: NewLongValueLookup(), + } self.currentTable = table self.Tables.Set(itemName, table) @@ -588,14 +640,67 @@ func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) error } self.currentTable.Indexes.Set(itemName, catalog) + case "CATALOG_TYPE_LONG_VALUE": + if Debug { + fmt.Printf("Catalog name %v for table %v\n", itemName, self.currentTable.Name) + } + lv := catalog.LongValue() + + WalkPages(self.ctx, int64(lv.FatherDataPageNumber()), + func(header *PageHeader, id int64, value *Value) error { + // Ignore tags that are too small to contain a key + if value.BufferSize < 8 { + return nil + } + + lv := self.ctx.Profile.LVKEY_BUFFER(value.reader, value.BufferOffset) + key := lv.ParseKey(self.ctx, header, value) + + long_value := &LongValue{ + Value: value, + header: header, + Key: key, + } + + self.currentTable.LongValueLookup[key.Key()] = long_value + + if Debug { + size := int(value.Tag._ValueSize()) + if size > 100 { + size = 100 + } + buffer := make([]byte, size) + value.Reader().ReadAt(buffer, 0) + + lv_buffer := long_value.Buffer() + if len(lv_buffer) > 100 { + lv_buffer = lv_buffer[:100] + } + fmt.Printf("------\nPage header %v\nID %v Tag %v\nPageID %v Flags %v\nKey %v \nLVBuffer %02x\nBuffer %02x \nTagLookup %v\n", + DebugPageHeader(self.ctx, header), id, + DebugTag(self.ctx, value.Tag, header), + value.PageID, + value.Flags, + long_value.Key.DebugString(), + lv_buffer, buffer, + len(self.currentTable.LongValueLookup)) + } + return nil + }) } return nil } -func (self *Catalog) Dump() string { +type DumpOptions struct { + LongValueTables bool + Indexes bool + Tables bool +} + +func (self *Catalog) Dump(options DumpOptions) string { result := "" for _, name := range self.Tables.Keys() { @@ -610,11 +715,40 @@ func (self *Catalog) Dump() string { column.Name, column.Type, column.Flags) } - result += fmt.Sprintf("%sIndexes\n", space) - for _, index := range table.Indexes.Keys() { - result += fmt.Sprintf("%s%s%v:\n", space, space, index) + if options.Indexes { + result += fmt.Sprintf("%sIndexes\n", space) + for _, index := range table.Indexes.Keys() { + result += fmt.Sprintf("%s%s%v:\n", space, space, index) + } + result += "\n" } - result += "\n" + + if options.LongValueTables && len(table.LongValueLookup) > 0 { + result += fmt.Sprintf("%sLongValues\n", space) + values := []*LongValue{} + for _, lv := range table.LongValueLookup { + values = append(values, lv) + } + + sort.Slice(values, func(i, j int) bool { + return values[i].Key.Key() < values[j].Key.Key() + }) + + for _, lv := range values { + buffer := lv.Buffer() + size := len(buffer) + if size > 100 { + buffer = buffer[:100] + } + result += fmt.Sprintf("%s%s%02x: \"%02x\"\n", + space, space, + lv.Key.Key(), + //lv.Key.DebugString(), + buffer) + } + result += "\n" + } + } return result diff --git a/context.go b/context.go index 19b642c..6332869 100644 --- a/context.go +++ b/context.go @@ -42,7 +42,10 @@ func NewESEContext(reader io.ReaderAt) (*ESEContext, error) { } func (self *ESEContext) GetPage(id int64) *PageHeader { - // First file page is file header, second page is backup of file header. - result := self.Profile.PageHeader(self.Reader, (id+1)*self.PageSize) - return result + // First file page is file header, second page is backup of file + // header. + return &PageHeader{ + PageHeader_: self.Profile.PageHeader_( + self.Reader, (id+1)*self.PageSize), + } } diff --git a/conversion.spec.yaml b/conversion.spec.yaml index e2f54d8..d1d40e7 100644 --- a/conversion.spec.yaml +++ b/conversion.spec.yaml @@ -3,11 +3,14 @@ Profile: ESEProfile Filename: ese_profile.json GenerateDebugString: true Structs: + # https://github.com/microsoft/Extensible-Storage-Engine/blob/933dc839b5a97b9a5b3e04824bdd456daf75a57d/dev/ese/src/inc/daedef.hxx#L2695 - FileHeader - DBTime - JET_LOGTIME - JET_SIGNATURE - - PageHeader + + # https://github.com/microsoft/Extensible-Storage-Engine/blob/933dc839b5a97b9a5b3e04824bdd456daf75a57d/dev/ese/src/inc/cpage.hxx#L887 + - PageHeader_ - Tag - ESENT_ROOT_HEADER - ESENT_BRANCH_HEADER @@ -26,6 +29,15 @@ Structs: - ESENT_CATALOG_DATA_DEFINITION_ENTRY - RecordTag - GUID + - LongValueHeader + + # https://github.com/microsoft/Extensible-Storage-Engine/blob/933dc839b5a97b9a5b3e04824bdd456daf75a57d/dev/ese/src/inc/lv.hxx#L37 + - LVKEY64 + - LVKEY32 + - LVKEY_BUFFER # This one is here to include some utility functions we might need. - Misc + + # some helpers + - CompressedKey diff --git a/debug.go b/debug.go index bbda069..c870900 100644 --- a/debug.go +++ b/debug.go @@ -1,5 +1,7 @@ package parser +import "fmt" + var ( // General purpose debug statements. Debug = false @@ -11,3 +13,13 @@ var ( func DlvDebug() { } + +func DebugPageHeader(ctx *ESEContext, page *PageHeader) string { + return page.DebugString() + fmt.Sprintf(" EndOffset: %#x \n", page.EndOffset(ctx)) +} + +func DebugTag(ctx *ESEContext, tag *Tag, page *PageHeader) string { + return tag.DebugString() + + fmt.Sprintf(" ValueOffsetInPage: %#x \n", + tag.ValueOffsetInPage(ctx, page)) +} diff --git a/ese_gen.go b/ese_gen.go index e788235..73906ad 100644 --- a/ese_gen.go +++ b/ese_gen.go @@ -97,29 +97,38 @@ type ESEProfile struct { Off_JET_LOGTIME_Year int64 Off_JET_SIGNATURE_Creation int64 Off_JET_SIGNATURE_CreatorMachine int64 + Off_LVKEY32_Lid int64 + Off_LVKEY32_SegmentOffset int64 + Off_LVKEY64_Lid int64 + Off_LVKEY64_SegmentOffset int64 + Off_LVKEY_BUFFER_PrefixLength int64 + Off_LVKEY_BUFFER_SuffixLength int64 + Off_LVKEY_BUFFER_KeyBuffer int64 Off_Misc_Misc int64 Off_Misc_Misc2 int64 Off_Misc_Misc3 int64 Off_Misc_Misc5 int64 Off_Misc_Misc4 int64 - Off_PageHeader_LastModified int64 - Off_PageHeader_PreviousPageNumber int64 - Off_PageHeader_NextPageNumber int64 - Off_PageHeader_FatherPage int64 - Off_PageHeader_AvailableDataSize int64 - Off_PageHeader_AvailableDataOffset int64 - Off_PageHeader_AvailablePageTag int64 - Off_PageHeader_Flags int64 + Off_PageHeader__LastModified int64 + Off_PageHeader__PreviousPageNumber int64 + Off_PageHeader__NextPageNumber int64 + Off_PageHeader__FatherPage int64 + Off_PageHeader__AvailableDataSize int64 + Off_PageHeader__AvailableDataOffset int64 + Off_PageHeader__AvailablePageTag int64 + Off_PageHeader__Flags int64 Off_RecordTag_Identifier int64 Off_RecordTag_DataOffset int64 Off_RecordTag_Flags int64 Off_Tag__ValueSize int64 Off_Tag__ValueOffset int64 + Off_Tag_Flags_ int64 + Off_Tag_Flags int64 } func NewESEProfile() *ESEProfile { // Specific offsets can be tweaked to cater for slight version mismatches. - self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,0,-2,0,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,4,6,8,0,1,2,3,4,5,4,12,0,0,0,0,0,8,16,20,24,28,32,34,36,0,2,2,0,2} + self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,0,-2,0,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,4,6,8,0,1,2,3,4,5,4,12,0,4,0,8,0,2,4,0,0,0,0,0,8,16,20,24,28,32,34,36,0,2,2,0,2,2,2} return self } @@ -199,12 +208,24 @@ func (self *ESEProfile) JET_SIGNATURE(reader io.ReaderAt, offset int64) *JET_SIG return &JET_SIGNATURE{Reader: reader, Offset: offset, Profile: self} } +func (self *ESEProfile) LVKEY32(reader io.ReaderAt, offset int64) *LVKEY32 { + return &LVKEY32{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) LVKEY64(reader io.ReaderAt, offset int64) *LVKEY64 { + return &LVKEY64{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) LVKEY_BUFFER(reader io.ReaderAt, offset int64) *LVKEY_BUFFER { + return &LVKEY_BUFFER{Reader: reader, Offset: offset, Profile: self} +} + func (self *ESEProfile) Misc(reader io.ReaderAt, offset int64) *Misc { return &Misc{Reader: reader, Offset: offset, Profile: self} } -func (self *ESEProfile) PageHeader(reader io.ReaderAt, offset int64) *PageHeader { - return &PageHeader{Reader: reader, Offset: offset, Profile: self} +func (self *ESEProfile) PageHeader_(reader io.ReaderAt, offset int64) *PageHeader_ { + return &PageHeader_{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) RecordTag(reader io.ReaderAt, offset int64) *RecordTag { @@ -548,8 +569,8 @@ func (self *ESENT_DATA_DEFINITION_HEADER) Size() int { return 4 } -func (self *ESENT_DATA_DEFINITION_HEADER) LastFixedType() int8 { - return ParseInt8(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_LastFixedType + self.Offset) +func (self *ESENT_DATA_DEFINITION_HEADER) LastFixedType() byte { + return ParseUint8(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_LastFixedType + self.Offset) } func (self *ESENT_DATA_DEFINITION_HEADER) LastVariableDataType() byte { @@ -601,8 +622,9 @@ func (self *ESENT_LEAF_ENTRY) CommonPageKeySize() uint16 { return ParseUint16(self.Reader, self.Profile.Off_ESENT_LEAF_ENTRY_CommonPageKeySize + self.Offset) } -func (self *ESENT_LEAF_ENTRY) LocalPageKeySize() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_ESENT_LEAF_ENTRY_LocalPageKeySize + self.Offset) +func (self *ESENT_LEAF_ENTRY) LocalPageKeySize() uint64 { + value := ParseUint16(self.Reader, self.Profile.Off_ESENT_LEAF_ENTRY_LocalPageKeySize + self.Offset) + return (uint64(value) & 0x1fff) >> 0x0 } func (self *ESENT_LEAF_ENTRY) DebugString() string { result := fmt.Sprintf("struct ESENT_LEAF_ENTRY @ %#x:\n", self.Offset) @@ -881,6 +903,84 @@ func (self *JET_SIGNATURE) DebugString() string { return result } +type LVKEY32 struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *LVKEY32) Size() int { + return 8 +} + +func (self *LVKEY32) Lid() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_LVKEY32_Lid + self.Offset) +} + +func (self *LVKEY32) SegmentOffset() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_LVKEY32_SegmentOffset + self.Offset) +} +func (self *LVKEY32) DebugString() string { + result := fmt.Sprintf("struct LVKEY32 @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Lid: %#0x\n", self.Lid()) + result += fmt.Sprintf(" SegmentOffset: %#0x\n", self.SegmentOffset()) + return result +} + +type LVKEY64 struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *LVKEY64) Size() int { + return 12 +} + +func (self *LVKEY64) Lid() uint64 { + return ParseUint64(self.Reader, self.Profile.Off_LVKEY64_Lid + self.Offset) +} + +func (self *LVKEY64) SegmentOffset() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_LVKEY64_SegmentOffset + self.Offset) +} +func (self *LVKEY64) DebugString() string { + result := fmt.Sprintf("struct LVKEY64 @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Lid: %#0x\n", self.Lid()) + result += fmt.Sprintf(" SegmentOffset: %#0x\n", self.SegmentOffset()) + return result +} + +type LVKEY_BUFFER struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *LVKEY_BUFFER) Size() int { + return 0 +} + +func (self *LVKEY_BUFFER) PrefixLength() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_LVKEY_BUFFER_PrefixLength + self.Offset) +} + +func (self *LVKEY_BUFFER) SuffixLength() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_LVKEY_BUFFER_SuffixLength + self.Offset) +} + + +func (self *LVKEY_BUFFER) KeyBuffer() string { + return ParseString(self.Reader, self.Profile.Off_LVKEY_BUFFER_KeyBuffer + self.Offset, 12) +} +func (self *LVKEY_BUFFER) DebugString() string { + result := fmt.Sprintf("struct LVKEY_BUFFER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" PrefixLength: %#0x\n", self.PrefixLength()) + result += fmt.Sprintf(" SuffixLength: %#0x\n", self.SuffixLength()) + result += fmt.Sprintf(" KeyBuffer: %v\n", string(self.KeyBuffer())) + return result +} + type Misc struct { Reader io.ReaderAt Offset int64 @@ -900,7 +1000,7 @@ func (self *Misc) Misc2() int16 { } func (self *Misc) Misc3() int64 { - return int64(ParseUint64(self.Reader, self.Profile.Off_Misc_Misc3 + self.Offset)) + return ParseInt64(self.Reader, self.Profile.Off_Misc_Misc3 + self.Offset) } func (self *Misc) Misc5() uint64 { @@ -921,46 +1021,46 @@ func (self *Misc) DebugString() string { return result } -type PageHeader struct { +type PageHeader_ struct { Reader io.ReaderAt Offset int64 Profile *ESEProfile } -func (self *PageHeader) Size() int { +func (self *PageHeader_) Size() int { return 0 } -func (self *PageHeader) LastModified() *DBTime { - return self.Profile.DBTime(self.Reader, self.Profile.Off_PageHeader_LastModified + self.Offset) +func (self *PageHeader_) LastModified() *DBTime { + return self.Profile.DBTime(self.Reader, self.Profile.Off_PageHeader__LastModified + self.Offset) } -func (self *PageHeader) PreviousPageNumber() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_PageHeader_PreviousPageNumber + self.Offset) +func (self *PageHeader_) PreviousPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_PageHeader__PreviousPageNumber + self.Offset) } -func (self *PageHeader) NextPageNumber() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_PageHeader_NextPageNumber + self.Offset) +func (self *PageHeader_) NextPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_PageHeader__NextPageNumber + self.Offset) } -func (self *PageHeader) FatherPage() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_PageHeader_FatherPage + self.Offset) +func (self *PageHeader_) FatherPage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_PageHeader__FatherPage + self.Offset) } -func (self *PageHeader) AvailableDataSize() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_PageHeader_AvailableDataSize + self.Offset) +func (self *PageHeader_) AvailableDataSize() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_PageHeader__AvailableDataSize + self.Offset) } -func (self *PageHeader) AvailableDataOffset() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_PageHeader_AvailableDataOffset + self.Offset) +func (self *PageHeader_) AvailableDataOffset() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_PageHeader__AvailableDataOffset + self.Offset) } -func (self *PageHeader) AvailablePageTag() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_PageHeader_AvailablePageTag + self.Offset) +func (self *PageHeader_) AvailablePageTag() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_PageHeader__AvailablePageTag + self.Offset) } -func (self *PageHeader) Flags() *Flags { - value := ParseUint32(self.Reader, self.Profile.Off_PageHeader_Flags + self.Offset) +func (self *PageHeader_) Flags() *Flags { + value := ParseUint32(self.Reader, self.Profile.Off_PageHeader__Flags + self.Offset) names := make(map[string]bool) @@ -995,8 +1095,8 @@ func (self *PageHeader) Flags() *Flags { return &Flags{Value: uint64(value), Names: names} } -func (self *PageHeader) DebugString() string { - result := fmt.Sprintf("struct PageHeader @ %#x:\n", self.Offset) +func (self *PageHeader_) DebugString() string { + result := fmt.Sprintf("struct PageHeader_ @ %#x:\n", self.Offset) result += fmt.Sprintf(" LastModified: {\n%v}\n", indent(self.LastModified().DebugString())) result += fmt.Sprintf(" PreviousPageNumber: %#0x\n", self.PreviousPageNumber()) result += fmt.Sprintf(" NextPageNumber: %#0x\n", self.NextPageNumber()) @@ -1046,7 +1146,7 @@ type Tag struct { } func (self *Tag) Size() int { - return 8 + return 4 } func (self *Tag) _ValueSize() uint16 { @@ -1056,10 +1156,37 @@ func (self *Tag) _ValueSize() uint16 { func (self *Tag) _ValueOffset() uint16 { return ParseUint16(self.Reader, self.Profile.Off_Tag__ValueOffset + self.Offset) } + +func (self *Tag) Flags_() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_Tag_Flags_ + self.Offset) +} + +func (self *Tag) Flags() *Flags { + value := ParseUint16(self.Reader, self.Profile.Off_Tag_Flags + self.Offset) + names := make(map[string]bool) + + + if value & 8192 != 0 { + names["fNDVersion"] = true + } + + if value & 16384 != 0 { + names["fNDDeleted"] = true + } + + if value & 32768 != 0 { + names["fNDCompressed"] = true + } + + return &Flags{Value: uint64(value), Names: names} +} + func (self *Tag) DebugString() string { result := fmt.Sprintf("struct Tag @ %#x:\n", self.Offset) result += fmt.Sprintf(" _ValueSize: %#0x\n", self._ValueSize()) result += fmt.Sprintf(" _ValueOffset: %#0x\n", self._ValueOffset()) + result += fmt.Sprintf(" Flags_: %#0x\n", self.Flags_()) + result += fmt.Sprintf(" Flags: %v\n", self.Flags().DebugString()) return result } @@ -1140,15 +1267,6 @@ func ParseInt64(reader io.ReaderAt, offset int64) int64 { return int64(binary.LittleEndian.Uint64(data)) } -func ParseInt8(reader io.ReaderAt, offset int64) int8 { - result := make([]byte, 1) - _, err := reader.ReadAt(result, offset) - if err != nil { - return 0 - } - return int8(result[0]) -} - func ParseUint16(reader io.ReaderAt, offset int64) uint16 { data := make([]byte, 2) _, err := reader.ReadAt(data, offset) @@ -1219,7 +1337,10 @@ func ParseTerminatedUTF16String(reader io.ReaderAt, offset int64) string { if idx < 0 { idx = n-1 } - return UTF16BytesToUTF8(data[0:idx+1], binary.LittleEndian) + if idx%2 != 0 { + idx += 1 + } + return UTF16BytesToUTF8(data[0:idx], binary.LittleEndian) } func ParseUTF16String(reader io.ReaderAt, offset int64, length int64) string { diff --git a/ese_profile.json b/ese_profile.json index eec31f3..319c04a 100644 --- a/ese_profile.json +++ b/ese_profile.json @@ -31,7 +31,7 @@ "Month": [4, ["unsigned char"]], "Year": [5, ["unsigned char"]] }], - "PageHeader": [0, { + "PageHeader_": [0, { "LastModified": [8, ["DBTime"]], "PreviousPageNumber": [16, ["unsigned long"]], "NextPageNumber": [20, ["unsigned long"]], @@ -52,9 +52,18 @@ } }]] }], - "Tag": [8, { + "Tag": [4, { "_ValueSize": [0, ["unsigned short"]], - "_ValueOffset": [2, ["unsigned short"]] + "_ValueOffset": [2, ["unsigned short"]], + "Flags_": [2, ["unsigned short"]], + "Flags": [2, ["Flags", { + "target": "unsigned short", + "maskmap": { + "fNDVersion": 8192, + "fNDDeleted": 16384, + "fNDCompressed": 32768 + } + }]] }], "ESENT_ROOT_HEADER": [16, { "InitialNumberOfPages": [0, ["unsigned long"]], @@ -92,7 +101,11 @@ "ESENT_LEAF_ENTRY": [16, { "CommonPageKeySize": [-2, ["unsigned short"]], - "LocalPageKeySize": [0, ["unsigned short"]] + "LocalPageKeySize": [0, ["BitField", { + "target": "unsigned short", + "start_bit": 0, + "end_bit": 13 + }]] }], "ESENT_BRANCH_ENTRY": [16, { @@ -156,7 +169,7 @@ }], "ESENT_DATA_DEFINITION_HEADER": [4, { - "LastFixedType": [0, ["char"]], + "LastFixedType": [0, ["unsigned char"]], "LastVariableDataType": [1, ["unsigned char"]], "VariableSizeOffset": [2, ["unsigned short"]] }], @@ -209,5 +222,23 @@ "count": 8, "target": "unsigned char" }]] + }], + + "LVKEY64": [12, { + "Lid": [0, ["unsigned long long"]], + "SegmentOffset": [8, ["unsigned long"]] + }], + + "LVKEY32": [8, { + "Lid": [0, ["unsigned long"]], + "SegmentOffset": [4, ["unsigned long"]] + }], + + "LVKEY_BUFFER": [0, { + "PrefixLength": [0, ["unsigned short"]], + "SuffixLength": [2, ["unsigned short"]], + "KeyBuffer": [4, ["String", { + "length": 12 + }]] }] } diff --git a/long_values.go b/long_values.go new file mode 100644 index 0000000..6d2b3b2 --- /dev/null +++ b/long_values.go @@ -0,0 +1,130 @@ +package parser + +import ( + "fmt" + "io" +) + +type LongValue struct { + Value *Value + header *PageHeader + + // Calculated key for this long value object. + Key Key +} + +func (self *LongValue) Buffer() []byte { + start := int64(self.Key.EndOffset()) + result := make([]byte, self.Value.BufferSize-start) + self.Value.Reader().ReadAt(result, start) + return result +} + +func (self *LongValue) Reader() io.ReaderAt { + start := int64(self.Key.EndOffset()) + return NewOffsetReader( + self.Value.Reader(), start, self.Value.BufferSize-start) +} + +type LongValueLookup map[string]*LongValue + +// This can potentially return a lot of data +func (self LongValueLookup) GetLid(lid []byte) ([]byte, bool) { + if len(lid) != 4 { + return nil, false + } + + // Swap byte order between Lid and key + swapped := []byte{lid[3], lid[2], lid[1], lid[0]} + + // For now we only get the first segment + key := Key{ + prefix: swapped, + suffix: make([]byte, 4), + } + + // Try to find segments + value, pres := self[key.Key()] + if pres { + return value.Buffer(), true + } + return nil, false +} + +func NewLongValueLookup() LongValueLookup { + return make(LongValueLookup) +} + +type Key struct { + prefix []byte + suffix []byte + + end_offset uint64 +} + +func (self *Key) Key() string { + return string(self.prefix) + string(self.suffix) +} + +func (self *Key) DebugString() string { + result := "" + if len(self.prefix) > 0 { + result += fmt.Sprintf("prefix %02x ", self.prefix) + } + + if len(self.suffix) > 0 { + result += fmt.Sprintf("suffix %02x ", self.suffix) + } + return result + fmt.Sprintf(" key %02x ", self.Key()) +} + +func (self *Key) EndOffset() uint64 { + return self.end_offset +} + +func (self *LVKEY_BUFFER) ParseKey(ctx *ESEContext, header *PageHeader, value *Value) (key Key) { + key.end_offset = uint64(ctx.Profile.Off_LVKEY_BUFFER_KeyBuffer) + + prefix_lenth := uint64(self.PrefixLength()) + if prefix_lenth > 8 { + prefix_lenth = 8 + } + + suffix_length := uint64(self.SuffixLength()) + if suffix_length > 8-prefix_lenth { + suffix_length = 8 - prefix_lenth + } + + // Compressed keys + if value.Tag.Flags_()&fNDCompressed > 0 { + external_value := header.ExternalValueBytes(ctx) + if prefix_lenth > uint64(len(external_value)) { + prefix_lenth = uint64(len(external_value)) + } + + for i := uint64(0); i < prefix_lenth; i++ { + key.prefix = append(key.prefix, external_value[i]) + } + key_buffer := self.KeyBuffer() + if suffix_length > 8 { + suffix_length = 8 + } + + key.suffix = []byte(key_buffer[:suffix_length]) + key.end_offset += suffix_length + + } else { + // The key is not compressed - we read both prefix + // and suffix from the actual key + key_buffer := []byte(self.KeyBuffer()) + for uint64(len(key_buffer)) < prefix_lenth+suffix_length { + key_buffer = append(key_buffer, 0) + } + + key.prefix = key_buffer[:prefix_lenth] + key.suffix = key_buffer[prefix_lenth : prefix_lenth+suffix_length] + key.end_offset += suffix_length + prefix_lenth + } + + return key +} diff --git a/pages.go b/pages.go index 1eb86fb..476296d 100644 --- a/pages.go +++ b/pages.go @@ -9,21 +9,40 @@ const ( TAG_COMMON = 4 ) +// TODO: This is called LINE in the MS code. It represents a single +// node in the page. Depending on the page type it needs to be further +// parsed. type Value struct { Tag *Tag PageID int64 - Buffer []byte Flags uint64 + + reader io.ReaderAt + BufferOffset int64 + BufferSize int64 +} + +func (self *Value) GetBuffer() []byte { + result := make([]byte, self.BufferSize) + self.reader.ReadAt(result, self.BufferOffset) + return result } func (self *Value) Reader() io.ReaderAt { - return &BufferReaderAt{self.Buffer} + return NewOffsetReader(self.reader, + self.BufferOffset, self.BufferSize) } -func NewValue(ctx *ESEContext, tag *Tag, PageID int64, buffer []byte) *Value { - result := &Value{Tag: tag, PageID: PageID, Buffer: buffer} +func NewReaderValue(ctx *ESEContext, tag *Tag, PageID int64, + reader io.ReaderAt, start, length int64) *Value { + result := &Value{Tag: tag, PageID: PageID, reader: reader, + BufferOffset: start, BufferSize: length} if ctx.Version == 0x620 && ctx.Revision >= 17 && - ctx.PageSize > 8192 && len(buffer) > 0 { + ctx.PageSize > 8192 && length > 0 { + + buffer := make([]byte, 4) + reader.ReadAt(buffer, start) + result.Flags = uint64(buffer[1] >> 5) buffer[1] &= 0x1f } else { @@ -32,15 +51,26 @@ func NewValue(ctx *ESEContext, tag *Tag, PageID int64, buffer []byte) *Value { return result } -func (self *Tag) ValueOffset(ctx *ESEContext) uint16 { +func (self *Tag) valueOffset(ctx *ESEContext) uint16 { if ctx.Version == 0x620 && ctx.Revision >= 17 && ctx.PageSize > 8192 { return self._ValueOffset() & 0x7FFF } return self._ValueOffset() & 0x1FFF } +func (self *Tag) ValueOffsetInPage(ctx *ESEContext, page *PageHeader) int64 { + return int64(self.valueOffset(ctx)) + page.EndOffset(ctx) +} + +func (self *Tag) FFlags() uint16 { + // CPAGE::TAG::FFlags + // https://github.com/microsoft/Extensible-Storage-Engine/blob/933dc839b5a97b9a5b3e04824bdd456daf75a57d/dev/ese/src/ese/cpage.cxx#1212 + return (self._ValueOffset() & 0x1fff) >> 13 +} + func (self *Tag) ValueSize(ctx *ESEContext) uint16 { - if ctx.Version == 0x620 && ctx.Revision >= 17 && ctx.PageSize > 8192 { + if ctx.Version == 0x620 && ctx.Revision >= 17 && + !IsSmallPage(ctx.PageSize) { return self._ValueSize() & 0x7FFF } return self._ValueSize() & 0x1FFF @@ -49,17 +79,20 @@ func (self *Tag) ValueSize(ctx *ESEContext) uint16 { func GetPageValues(ctx *ESEContext, header *PageHeader, id int64) []*Value { result := []*Value{} - // Tags are written from the end of the page + // Tags are written from the end of the page. Sizeof(Tag) = 4 offset := ctx.PageSize + header.Offset - 4 + // Skip the external value tag because it is fetched using a + // dedicated call to PageHeader.ExternalValue() + offset -= 4 + for tag_count := header.AvailablePageTag(); tag_count > 0; tag_count-- { tag := ctx.Profile.Tag(ctx.Reader, offset) - value_offset := header.EndOffset(ctx) + int64(tag.ValueOffset(ctx)) - - buffer := make([]byte, int(tag.ValueSize(ctx))) - ctx.Reader.ReadAt(buffer, value_offset) - result = append(result, NewValue(ctx, tag, id, buffer)) + result = append(result, NewReaderValue( + ctx, tag, id, ctx.Reader, + tag.ValueOffsetInPage(ctx, header), + int64(tag.ValueSize(ctx)))) offset -= 4 } @@ -77,6 +110,33 @@ func GetBranch(ctx *ESEContext, value *Value) *ESENT_BRANCH_HEADER { return ctx.Profile.ESENT_BRANCH_HEADER(value.Reader(), 0) } +type PageHeader struct { + *PageHeader_ + + // The value pointed to by tag 0 + external_value_bytes []byte +} + +func (self *PageHeader) ExternalValueBytes(ctx *ESEContext) []byte { + if self.external_value_bytes != nil { + return self.external_value_bytes + } + + self.external_value_bytes = self.ExternalValue(ctx).GetBuffer() + return self.external_value_bytes +} + +// The External value is the zero'th tag +func (self *PageHeader) ExternalValue(ctx *ESEContext) *Value { + offset := ctx.PageSize + self.Offset - 4 + tag := self.Profile.Tag(self.Reader, offset) + + return NewReaderValue( + ctx, tag, 0, ctx.Reader, + tag.ValueOffsetInPage(ctx, self), + int64(tag.ValueSize(ctx))) +} + func (self *PageHeader) IsBranch() bool { return !self.Flags().IsSet("Leaf") } @@ -86,15 +146,14 @@ func (self *PageHeader) IsLeaf() bool { } func (self *PageHeader) EndOffset(ctx *ESEContext) int64 { - // Common size size := int64(40) - // Depending on version, the size of the header is different. - if ctx.Version == 0x620 && ctx.Revision >= 0x11 && ctx.PageSize > 8192 { - // Windows 7 and later - size += 5 * 8 + // The header is larger when the pagesize is bigger (PGHDR2 vs + // PGHDR) + // https://github.com/microsoft/Extensible-Storage-Engine/blob/933dc839b5a97b9a5b3e04824bdd456daf75a57d/dev/ese/src/inc/cpage.hxx#L885 + if !IsSmallPage(ctx.PageSize) { + size = 80 } - return self.Offset + size } @@ -110,7 +169,8 @@ func DumpPage(ctx *ESEContext, id int64) { for i, value := range values { fmt.Printf("Tag %v @ %#x offset %#x length %#x\n", - i, value.Tag.Offset, value.Tag.ValueOffset(ctx), + i, value.Tag.Offset, + value.Tag.ValueOffsetInPage(ctx, header), value.Tag.ValueSize(ctx)) } @@ -126,7 +186,7 @@ func DumpPage(ctx *ESEContext, id int64) { // SpaceTree header } else if flags.IsSet("SpaceTree") { ctx.Profile.ESENT_SPACE_TREE_HEADER( - &BufferReaderAt{values[0].Buffer}, 0).Dump() + ctx.Reader, values[0].BufferOffset).Dump() // Leaf header } else if header.IsLeaf() { @@ -238,21 +298,21 @@ func _walkPages(ctx *ESEContext, } seen[id] = true - if DebugWalk { - fmt.Printf("Walking page %v\n", id) - } - header := ctx.GetPage(id) values := GetPageValues(ctx, header, id) + if DebugWalk { + fmt.Printf("Walking page %v %v\n", id, header.DebugString()) + } // No more records. if len(values) == 0 { return nil } - for _, value := range values[1:] { + for _, value := range values { if header.IsLeaf() { - // Allow the callback to return early (e.g. in case of cancellation) + // Allow the callback to return early (e.g. in case of + // cancellation) err := cb(header, id, value) if err != nil { return err diff --git a/reader.go b/reader.go index c2444ee..4e18ad0 100644 --- a/reader.go +++ b/reader.go @@ -1,5 +1,7 @@ package parser +import "io" + type BufferReaderAt struct { buffer []byte } @@ -23,3 +25,29 @@ func (self *BufferReaderAt) ReadAt(buf []byte, offset int64) (int, error) { return n, nil } + +type OffsetReader struct { + reader io.ReaderAt + offset int64 + length int64 +} + +func (self OffsetReader) ReadAt(buff []byte, off int64) (int, error) { + to_read := int64(len(buff)) + if off+to_read > self.length { + to_read = self.length - off + } + + if to_read < 0 { + return 0, nil + } + return self.reader.ReadAt(buff, off+self.offset) +} + +func NewOffsetReader(reader io.ReaderAt, offset, size int64) io.ReaderAt { + return &OffsetReader{ + reader: reader, + offset: offset, + length: offset + size, + } +} diff --git a/utils.go b/utils.go index 25128e1..5d820c7 100644 --- a/utils.go +++ b/utils.go @@ -9,3 +9,7 @@ func WinFileTime64(reader io.ReaderAt, offset int64) time.Time { value := ParseInt64(reader, offset) return time.Unix((value/10000000)-11644473600, 0).UTC() } + +func IsSmallPage(page_size int64) bool { + return page_size <= 1024*8 +} From 18bc6dc22bbb623b7f0a6c5fc5eaad72fb61d7c2 Mon Sep 17 00:00:00 2001 From: Mike Cohen Date: Wed, 7 Feb 2024 10:54:44 +1000 Subject: [PATCH 38/40] Bugfix: Off by one on parsing tags can return garbage LINE (#23) --- pages.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages.go b/pages.go index 476296d..5796484 100644 --- a/pages.go +++ b/pages.go @@ -86,7 +86,7 @@ func GetPageValues(ctx *ESEContext, header *PageHeader, id int64) []*Value { // dedicated call to PageHeader.ExternalValue() offset -= 4 - for tag_count := header.AvailablePageTag(); tag_count > 0; tag_count-- { + for tag_count := header.AvailablePageTag() - 1; tag_count > 0; tag_count-- { tag := ctx.Profile.Tag(ctx.Reader, offset) result = append(result, NewReaderValue( From cfb0ba4ad963f5989c8ed9b1a091c62b888bd847 Mon Sep 17 00:00:00 2001 From: Simon Lehn <48837958+srlehn@users.noreply.github.com> Date: Sat, 1 Jun 2024 00:20:12 +0200 Subject: [PATCH 39/40] create internal fork of www.velocidex.com/golang/go-ese/parser and github.com/Velocidex/ordereddict with additional json and yaml dependencies stripped off --- go.mod | 5 - go.sum | 45 - internal/eseparser/catalog.go | 37 +- internal/eseparser/compression.go | 2 +- internal/eseparser/context.go | 2 +- internal/eseparser/debug.go | 2 +- internal/eseparser/ese_gen.go | 1367 ++++++++--------- internal/eseparser/guid.go | 2 +- internal/eseparser/long_values.go | 2 +- internal/eseparser/ordereddict/Makefile | 2 - internal/eseparser/ordereddict/README.md | 2 - internal/eseparser/ordereddict/go.mod | 10 - internal/eseparser/ordereddict/go.sum | 23 - internal/eseparser/ordereddict/ordereddict.go | 267 +--- .../eseparser/ordereddict/ordereddict_test.go | 148 -- internal/eseparser/pages.go | 2 +- internal/eseparser/reader.go | 2 +- internal/eseparser/utils.go | 2 +- internal/ie/cookiestore.go | 9 +- internal/ie/ese.go | 8 +- 20 files changed, 698 insertions(+), 1241 deletions(-) delete mode 100644 internal/eseparser/ordereddict/Makefile delete mode 100644 internal/eseparser/ordereddict/README.md delete mode 100644 internal/eseparser/ordereddict/go.mod delete mode 100644 internal/eseparser/ordereddict/go.sum delete mode 100644 internal/eseparser/ordereddict/ordereddict_test.go diff --git a/go.mod b/go.mod index 36f9edb..a17f3b2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/browserutils/kooky go 1.18 require ( - github.com/Velocidex/ordereddict v0.0.0-20230909174157-2aa49cc5d11d github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 github.com/go-ini/ini v1.67.0 github.com/go-sqlite/sqlite3 v0.0.0-20180313105335-53dd8e640ee7 @@ -14,13 +13,9 @@ require ( golang.org/x/net v0.25.0 golang.org/x/sys v0.20.0 golang.org/x/text v0.15.0 - www.velocidex.com/golang/go-ese v0.2.0 ) require ( - github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a // indirect - github.com/Velocidex/yaml/v2 v2.2.8 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gonuts/binary v0.2.0 // indirect ) diff --git a/go.sum b/go.sum index f55be11..26a8098 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,5 @@ -github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a h1:AeXPUzhU0yhID/v5JJEIkjaE85ASe+Vh4Kuv1RSLL+4= -github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a/go.mod h1:ukJBuruT9b24pdgZwWDvOaCYHeS03B7oQPCUWh25bwM= -github.com/Velocidex/ordereddict v0.0.0-20220107075049-3dbe58412844/go.mod h1:Y5Tfx5SKGOzkulpqfonrdILSPIuNg+GqKE/DhVJgnpg= -github.com/Velocidex/ordereddict v0.0.0-20230909174157-2aa49cc5d11d h1:fn372EqKyazBxYUP5HPpBi3jId4MXuppEypEALGfvEk= -github.com/Velocidex/ordereddict v0.0.0-20230909174157-2aa49cc5d11d/go.mod h1:+MqO5UMBemyFSm+yRXslbpFTwPUDhFHUf7HPV92twg4= -github.com/Velocidex/yaml/v2 v2.2.8 h1:GUrSy4SBJ6RjGt43k6MeBKtw2z/27gh4A3hfFmFY3No= -github.com/Velocidex/yaml/v2 v2.2.8/go.mod h1:PlXIg/Pxmoja48C1vMHo7C5pauAZvLq/UEPOQ3DsjS4= -github.com/alecthomas/assert v1.0.0/go.mod h1:va/d2JC+M7F6s+80kl/R3G7FUiW6JzUO+hPhLyJ36ZY= -github.com/alecthomas/colour v0.1.0/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= -github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= -github.com/alecthomas/repr v0.1.1 h1:87P60cSmareLAxMc4Hro0r2RBY4ROm0dYwkJNpS4pPs= -github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 h1:2pkAuIM8OF1fy4ToFpMnI4oE+VeUNRbGrpSLKshK0oQ= github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89/go.mod h1:/09nEjna1UMoasyyQDhOrIn8hi2v2kiJglPWed1idck= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= @@ -27,28 +12,11 @@ github.com/gonuts/binary v0.2.0 h1:caITwMWAoQWlL0RNvv2lTU/AHqAJlVuu6nZmNgfbKW4= github.com/gonuts/binary v0.2.0/go.mod h1:kM+CtBrCGDSKdv8WXTuCUsw+loiy8f/QEI8YCCC0M/E= github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs= github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/zalando/go-keyring v0.2.4 h1:wi2xxTqdiwMKbM6TWwi+uJCG/Tum2UV0jqaQhCa9/68= github.com/zalando/go-keyring v0.2.4/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= @@ -57,22 +25,9 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -www.velocidex.com/golang/go-ese v0.2.0 h1:8/hzEMupfqEF0oMi1/EzsMN1xLN0GBFcB3GqxqRnb9s= -www.velocidex.com/golang/go-ese v0.2.0/go.mod h1:6fC9T6UGLbM7icuA0ugomU5HbFC5XA5I30zlWtZT8YE= diff --git a/internal/eseparser/catalog.go b/internal/eseparser/catalog.go index 597e35f..9efff11 100644 --- a/internal/eseparser/catalog.go +++ b/internal/eseparser/catalog.go @@ -1,6 +1,6 @@ // Parser based on https://github.com/SecureAuthCorp/impacket.git -package parser +package eseparser import ( "encoding/hex" @@ -10,8 +10,7 @@ import ( "sort" "time" - "github.com/Velocidex/ordereddict" - "github.com/davecgh/go-spew/spew" + "github.com/browserutils/kooky/internal/eseparser/ordereddict" ) const ( @@ -82,14 +81,6 @@ type Table struct { // Column RDomain Identifier 256 Type Long Text func (self *Table) tagToRecord(value *Value, header *PageHeader) *ordereddict.Dict { - if Debug { - fmt.Printf("Processing row in Tag @ %d %#x (%#x)", - value.Tag.Offset, - value.Tag.ValueOffsetInPage(self.ctx, header), - value.Tag.ValueSize(self.ctx)) - spew.Dump(value.GetBuffer()) - } - result := ordereddict.NewDict() var taggedItems map[uint32][]byte @@ -467,22 +458,22 @@ type tagBuffer struct { } /* - Tagged values are used to store sparse values. +Tagged values are used to store sparse values. - They consist of an array of RecordTag, each RecordTag has an - Identifier and an offset to the start of its data. The length of the - data in each record is determine by the start of the next record. +They consist of an array of RecordTag, each RecordTag has an +Identifier and an offset to the start of its data. The length of the +data in each record is determine by the start of the next record. - Example: +Example: - 00000050 00 01 0c 40 a4 01 21 00 a5 01 23 00 01 6c 00 61 |...@..!...#..l.a| - 00000060 00 62 00 5c 00 64 00 63 00 2d 00 31 00 24 00 00 |.b.\.d.c.-.1.$..| - 00000070 00 3d 00 f9 00 |.=...| +00000050 00 01 0c 40 a4 01 21 00 a5 01 23 00 01 6c 00 61 |...@..!...#..l.a| +00000060 00 62 00 5c 00 64 00 63 00 2d 00 31 00 24 00 00 |.b.\.d.c.-.1.$..| +00000070 00 3d 00 f9 00 |.=...| - Slice is 0x50-0x75 00010c40a4012100a5012300016c00610062005c00640063002d003100240000003d00f900 - Consumed 0x15 bytes of TAGGED space from 0xc to 0x21 for tag 0x100 - Consumed 0x2 bytes of TAGGED space from 0x21 to 0x23 for tag 0x1a4 - Consumed 0x2 bytes of TAGGED space from 0x23 to 0x25 for tag 0x1a5 +Slice is 0x50-0x75 00010c40a4012100a5012300016c00610062005c00640063002d003100240000003d00f900 +Consumed 0x15 bytes of TAGGED space from 0xc to 0x21 for tag 0x100 +Consumed 0x2 bytes of TAGGED space from 0x21 to 0x23 for tag 0x1a4 +Consumed 0x2 bytes of TAGGED space from 0x23 to 0x25 for tag 0x1a5 */ func ParseTaggedValues(ctx *ESEContext, buffer []byte) map[uint32][]byte { result := make(map[uint32][]byte) diff --git a/internal/eseparser/compression.go b/internal/eseparser/compression.go index 0606c7d..0f4bb8a 100644 --- a/internal/eseparser/compression.go +++ b/internal/eseparser/compression.go @@ -1,4 +1,4 @@ -package parser +package eseparser import ( "encoding/binary" diff --git a/internal/eseparser/context.go b/internal/eseparser/context.go index 6332869..9f4bf62 100644 --- a/internal/eseparser/context.go +++ b/internal/eseparser/context.go @@ -1,4 +1,4 @@ -package parser +package eseparser import ( "errors" diff --git a/internal/eseparser/debug.go b/internal/eseparser/debug.go index c870900..af2b561 100644 --- a/internal/eseparser/debug.go +++ b/internal/eseparser/debug.go @@ -1,4 +1,4 @@ -package parser +package eseparser import "fmt" diff --git a/internal/eseparser/ese_gen.go b/internal/eseparser/ese_gen.go index 73906ad..d8db5e1 100644 --- a/internal/eseparser/ese_gen.go +++ b/internal/eseparser/ese_gen.go @@ -1,1355 +1,1324 @@ - -package parser +package eseparser // Autogenerated code from ese_profile.json. Do not edit. import ( - "encoding/binary" - "fmt" - "bytes" - "io" - "sort" - "strings" - "unicode/utf16" - "unicode/utf8" -) - -var ( - // Depending on autogenerated code we may use this. Add a reference - // to shut the compiler up. - _ = bytes.MinRead - _ = fmt.Sprintf - _ = utf16.Decode - _ = binary.LittleEndian - _ = utf8.RuneError - _ = sort.Strings - _ = strings.Join - _ = io.Copy + "bytes" + "encoding/binary" + "fmt" + "io" + "sort" + "strings" + "unicode/utf16" + "unicode/utf8" ) func indent(text string) string { - result := []string{} - lines := strings.Split(text,"\n") - for _, line := range lines { - result = append(result, " " + line) - } - return strings.Join(result, "\n") + result := []string{} + lines := strings.Split(text, "\n") + for _, line := range lines { + result = append(result, " "+line) + } + return strings.Join(result, "\n") } - type ESEProfile struct { - Off_CATALOG_TYPE_COLUMN_ColumnType int64 - Off_CATALOG_TYPE_COLUMN_SpaceUsage int64 - Off_CATALOG_TYPE_COLUMN_ColumnFlags int64 - Off_CATALOG_TYPE_COLUMN_CodePage int64 - Off_CATALOG_TYPE_INDEX_FatherDataPageNumber int64 - Off_CATALOG_TYPE_INDEX_SpaceUsage int64 - Off_CATALOG_TYPE_INDEX_IndexFlags int64 - Off_CATALOG_TYPE_INDEX_Locale int64 - Off_CATALOG_TYPE_LONG_VALUE_FatherDataPageNumber int64 - Off_CATALOG_TYPE_LONG_VALUE_SpaceUsage int64 - Off_CATALOG_TYPE_LONG_VALUE_LVFlags int64 - Off_CATALOG_TYPE_LONG_VALUE_InitialNumberOfPages int64 - Off_CATALOG_TYPE_TABLE_FatherDataPageNumber int64 - Off_CATALOG_TYPE_TABLE_SpaceUsage int64 - Off_DBTime_Hours int64 - Off_DBTime_Min int64 - Off_DBTime_Sec int64 - Off_ESENT_BRANCH_ENTRY_LocalPageKeySize int64 - Off_ESENT_BRANCH_HEADER_CommonPageKey int64 - Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_FDPId int64 - Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Type int64 - Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Identifier int64 - Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Column int64 - Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Table int64 - Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Index int64 - Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_LongValue int64 - Off_ESENT_DATA_DEFINITION_HEADER_LastFixedType int64 - Off_ESENT_DATA_DEFINITION_HEADER_LastVariableDataType int64 - Off_ESENT_DATA_DEFINITION_HEADER_VariableSizeOffset int64 - Off_ESENT_INDEX_ENTRY_RecordPageKey int64 - Off_ESENT_LEAF_ENTRY_CommonPageKeySize int64 - Off_ESENT_LEAF_ENTRY_LocalPageKeySize int64 - Off_ESENT_LEAF_HEADER_CommonPageKey int64 - Off_ESENT_ROOT_HEADER_InitialNumberOfPages int64 - Off_ESENT_ROOT_HEADER_ParentFDP int64 - Off_ESENT_ROOT_HEADER_ExtentSpace int64 - Off_ESENT_ROOT_HEADER_SpaceTreePageNumber int64 - Off_ESENT_SPACE_TREE_ENTRY_PageKeySize int64 - Off_ESENT_SPACE_TREE_ENTRY_LastPageNumber int64 - Off_ESENT_SPACE_TREE_ENTRY_NumberOfPages int64 - Off_FileHeader_Magic int64 - Off_FileHeader_FormatVersion int64 - Off_FileHeader_FormatRevision int64 - Off_FileHeader_FileType int64 - Off_FileHeader_DataBaseTime int64 - Off_FileHeader_Signature int64 - Off_FileHeader_PageSize int64 - Off_GUID_Data1 int64 - Off_GUID_Data2 int64 - Off_GUID_Data3 int64 - Off_GUID_Data4 int64 - Off_JET_LOGTIME_Sec int64 - Off_JET_LOGTIME_Min int64 - Off_JET_LOGTIME_Hours int64 - Off_JET_LOGTIME_Days int64 - Off_JET_LOGTIME_Month int64 - Off_JET_LOGTIME_Year int64 - Off_JET_SIGNATURE_Creation int64 - Off_JET_SIGNATURE_CreatorMachine int64 - Off_LVKEY32_Lid int64 - Off_LVKEY32_SegmentOffset int64 - Off_LVKEY64_Lid int64 - Off_LVKEY64_SegmentOffset int64 - Off_LVKEY_BUFFER_PrefixLength int64 - Off_LVKEY_BUFFER_SuffixLength int64 - Off_LVKEY_BUFFER_KeyBuffer int64 - Off_Misc_Misc int64 - Off_Misc_Misc2 int64 - Off_Misc_Misc3 int64 - Off_Misc_Misc5 int64 - Off_Misc_Misc4 int64 - Off_PageHeader__LastModified int64 - Off_PageHeader__PreviousPageNumber int64 - Off_PageHeader__NextPageNumber int64 - Off_PageHeader__FatherPage int64 - Off_PageHeader__AvailableDataSize int64 - Off_PageHeader__AvailableDataOffset int64 - Off_PageHeader__AvailablePageTag int64 - Off_PageHeader__Flags int64 - Off_RecordTag_Identifier int64 - Off_RecordTag_DataOffset int64 - Off_RecordTag_Flags int64 - Off_Tag__ValueSize int64 - Off_Tag__ValueOffset int64 - Off_Tag_Flags_ int64 - Off_Tag_Flags int64 + Off_CATALOG_TYPE_COLUMN_ColumnType int64 + Off_CATALOG_TYPE_COLUMN_SpaceUsage int64 + Off_CATALOG_TYPE_COLUMN_ColumnFlags int64 + Off_CATALOG_TYPE_COLUMN_CodePage int64 + Off_CATALOG_TYPE_INDEX_FatherDataPageNumber int64 + Off_CATALOG_TYPE_INDEX_SpaceUsage int64 + Off_CATALOG_TYPE_INDEX_IndexFlags int64 + Off_CATALOG_TYPE_INDEX_Locale int64 + Off_CATALOG_TYPE_LONG_VALUE_FatherDataPageNumber int64 + Off_CATALOG_TYPE_LONG_VALUE_SpaceUsage int64 + Off_CATALOG_TYPE_LONG_VALUE_LVFlags int64 + Off_CATALOG_TYPE_LONG_VALUE_InitialNumberOfPages int64 + Off_CATALOG_TYPE_TABLE_FatherDataPageNumber int64 + Off_CATALOG_TYPE_TABLE_SpaceUsage int64 + Off_DBTime_Hours int64 + Off_DBTime_Min int64 + Off_DBTime_Sec int64 + Off_ESENT_BRANCH_ENTRY_LocalPageKeySize int64 + Off_ESENT_BRANCH_HEADER_CommonPageKey int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_FDPId int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Type int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Identifier int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Column int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Table int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Index int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_LongValue int64 + Off_ESENT_DATA_DEFINITION_HEADER_LastFixedType int64 + Off_ESENT_DATA_DEFINITION_HEADER_LastVariableDataType int64 + Off_ESENT_DATA_DEFINITION_HEADER_VariableSizeOffset int64 + Off_ESENT_INDEX_ENTRY_RecordPageKey int64 + Off_ESENT_LEAF_ENTRY_CommonPageKeySize int64 + Off_ESENT_LEAF_ENTRY_LocalPageKeySize int64 + Off_ESENT_LEAF_HEADER_CommonPageKey int64 + Off_ESENT_ROOT_HEADER_InitialNumberOfPages int64 + Off_ESENT_ROOT_HEADER_ParentFDP int64 + Off_ESENT_ROOT_HEADER_ExtentSpace int64 + Off_ESENT_ROOT_HEADER_SpaceTreePageNumber int64 + Off_ESENT_SPACE_TREE_ENTRY_PageKeySize int64 + Off_ESENT_SPACE_TREE_ENTRY_LastPageNumber int64 + Off_ESENT_SPACE_TREE_ENTRY_NumberOfPages int64 + Off_FileHeader_Magic int64 + Off_FileHeader_FormatVersion int64 + Off_FileHeader_FormatRevision int64 + Off_FileHeader_FileType int64 + Off_FileHeader_DataBaseTime int64 + Off_FileHeader_Signature int64 + Off_FileHeader_PageSize int64 + Off_GUID_Data1 int64 + Off_GUID_Data2 int64 + Off_GUID_Data3 int64 + Off_GUID_Data4 int64 + Off_JET_LOGTIME_Sec int64 + Off_JET_LOGTIME_Min int64 + Off_JET_LOGTIME_Hours int64 + Off_JET_LOGTIME_Days int64 + Off_JET_LOGTIME_Month int64 + Off_JET_LOGTIME_Year int64 + Off_JET_SIGNATURE_Creation int64 + Off_JET_SIGNATURE_CreatorMachine int64 + Off_LVKEY32_Lid int64 + Off_LVKEY32_SegmentOffset int64 + Off_LVKEY64_Lid int64 + Off_LVKEY64_SegmentOffset int64 + Off_LVKEY_BUFFER_PrefixLength int64 + Off_LVKEY_BUFFER_SuffixLength int64 + Off_LVKEY_BUFFER_KeyBuffer int64 + Off_Misc_Misc int64 + Off_Misc_Misc2 int64 + Off_Misc_Misc3 int64 + Off_Misc_Misc5 int64 + Off_Misc_Misc4 int64 + Off_PageHeader__LastModified int64 + Off_PageHeader__PreviousPageNumber int64 + Off_PageHeader__NextPageNumber int64 + Off_PageHeader__FatherPage int64 + Off_PageHeader__AvailableDataSize int64 + Off_PageHeader__AvailableDataOffset int64 + Off_PageHeader__AvailablePageTag int64 + Off_PageHeader__Flags int64 + Off_RecordTag_Identifier int64 + Off_RecordTag_DataOffset int64 + Off_RecordTag_Flags int64 + Off_Tag__ValueSize int64 + Off_Tag__ValueOffset int64 + Off_Tag_Flags_ int64 + Off_Tag_Flags int64 } func NewESEProfile() *ESEProfile { - // Specific offsets can be tweaked to cater for slight version mismatches. - self := &ESEProfile{0,4,8,12,0,4,8,12,0,4,8,12,0,4,0,2,4,0,0,0,4,6,10,10,10,10,0,1,2,0,-2,0,0,0,4,8,12,0,0,0,4,8,232,12,16,24,236,0,4,6,8,0,1,2,3,4,5,4,12,0,4,0,8,0,2,4,0,0,0,0,0,8,16,20,24,28,32,34,36,0,2,2,0,2,2,2} - return self + // Specific offsets can be tweaked to cater for slight version mismatches. + self := &ESEProfile{0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 0, 2, 4, 0, 0, 0, 4, 6, 10, 10, 10, 10, 0, 1, 2, 0, -2, 0, 0, 0, 4, 8, 12, 0, 0, 0, 4, 8, 232, 12, 16, 24, 236, 0, 4, 6, 8, 0, 1, 2, 3, 4, 5, 4, 12, 0, 4, 0, 8, 0, 2, 4, 0, 0, 0, 0, 0, 8, 16, 20, 24, 28, 32, 34, 36, 0, 2, 2, 0, 2, 2, 2} + return self } func (self *ESEProfile) CATALOG_TYPE_COLUMN(reader io.ReaderAt, offset int64) *CATALOG_TYPE_COLUMN { - return &CATALOG_TYPE_COLUMN{Reader: reader, Offset: offset, Profile: self} + return &CATALOG_TYPE_COLUMN{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) CATALOG_TYPE_INDEX(reader io.ReaderAt, offset int64) *CATALOG_TYPE_INDEX { - return &CATALOG_TYPE_INDEX{Reader: reader, Offset: offset, Profile: self} + return &CATALOG_TYPE_INDEX{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) CATALOG_TYPE_LONG_VALUE(reader io.ReaderAt, offset int64) *CATALOG_TYPE_LONG_VALUE { - return &CATALOG_TYPE_LONG_VALUE{Reader: reader, Offset: offset, Profile: self} + return &CATALOG_TYPE_LONG_VALUE{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) CATALOG_TYPE_TABLE(reader io.ReaderAt, offset int64) *CATALOG_TYPE_TABLE { - return &CATALOG_TYPE_TABLE{Reader: reader, Offset: offset, Profile: self} + return &CATALOG_TYPE_TABLE{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) DBTime(reader io.ReaderAt, offset int64) *DBTime { - return &DBTime{Reader: reader, Offset: offset, Profile: self} + return &DBTime{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) ESENT_BRANCH_ENTRY(reader io.ReaderAt, offset int64) *ESENT_BRANCH_ENTRY { - return &ESENT_BRANCH_ENTRY{Reader: reader, Offset: offset, Profile: self} + return &ESENT_BRANCH_ENTRY{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) ESENT_BRANCH_HEADER(reader io.ReaderAt, offset int64) *ESENT_BRANCH_HEADER { - return &ESENT_BRANCH_HEADER{Reader: reader, Offset: offset, Profile: self} + return &ESENT_BRANCH_HEADER{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) ESENT_CATALOG_DATA_DEFINITION_ENTRY(reader io.ReaderAt, offset int64) *ESENT_CATALOG_DATA_DEFINITION_ENTRY { - return &ESENT_CATALOG_DATA_DEFINITION_ENTRY{Reader: reader, Offset: offset, Profile: self} + return &ESENT_CATALOG_DATA_DEFINITION_ENTRY{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) ESENT_DATA_DEFINITION_HEADER(reader io.ReaderAt, offset int64) *ESENT_DATA_DEFINITION_HEADER { - return &ESENT_DATA_DEFINITION_HEADER{Reader: reader, Offset: offset, Profile: self} + return &ESENT_DATA_DEFINITION_HEADER{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) ESENT_INDEX_ENTRY(reader io.ReaderAt, offset int64) *ESENT_INDEX_ENTRY { - return &ESENT_INDEX_ENTRY{Reader: reader, Offset: offset, Profile: self} + return &ESENT_INDEX_ENTRY{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) ESENT_LEAF_ENTRY(reader io.ReaderAt, offset int64) *ESENT_LEAF_ENTRY { - return &ESENT_LEAF_ENTRY{Reader: reader, Offset: offset, Profile: self} + return &ESENT_LEAF_ENTRY{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) ESENT_LEAF_HEADER(reader io.ReaderAt, offset int64) *ESENT_LEAF_HEADER { - return &ESENT_LEAF_HEADER{Reader: reader, Offset: offset, Profile: self} + return &ESENT_LEAF_HEADER{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) ESENT_ROOT_HEADER(reader io.ReaderAt, offset int64) *ESENT_ROOT_HEADER { - return &ESENT_ROOT_HEADER{Reader: reader, Offset: offset, Profile: self} + return &ESENT_ROOT_HEADER{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) ESENT_SPACE_TREE_ENTRY(reader io.ReaderAt, offset int64) *ESENT_SPACE_TREE_ENTRY { - return &ESENT_SPACE_TREE_ENTRY{Reader: reader, Offset: offset, Profile: self} + return &ESENT_SPACE_TREE_ENTRY{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) ESENT_SPACE_TREE_HEADER(reader io.ReaderAt, offset int64) *ESENT_SPACE_TREE_HEADER { - return &ESENT_SPACE_TREE_HEADER{Reader: reader, Offset: offset, Profile: self} + return &ESENT_SPACE_TREE_HEADER{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) FileHeader(reader io.ReaderAt, offset int64) *FileHeader { - return &FileHeader{Reader: reader, Offset: offset, Profile: self} + return &FileHeader{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) GUID(reader io.ReaderAt, offset int64) *GUID { - return &GUID{Reader: reader, Offset: offset, Profile: self} + return &GUID{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) JET_LOGTIME(reader io.ReaderAt, offset int64) *JET_LOGTIME { - return &JET_LOGTIME{Reader: reader, Offset: offset, Profile: self} + return &JET_LOGTIME{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) JET_SIGNATURE(reader io.ReaderAt, offset int64) *JET_SIGNATURE { - return &JET_SIGNATURE{Reader: reader, Offset: offset, Profile: self} + return &JET_SIGNATURE{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) LVKEY32(reader io.ReaderAt, offset int64) *LVKEY32 { - return &LVKEY32{Reader: reader, Offset: offset, Profile: self} + return &LVKEY32{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) LVKEY64(reader io.ReaderAt, offset int64) *LVKEY64 { - return &LVKEY64{Reader: reader, Offset: offset, Profile: self} + return &LVKEY64{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) LVKEY_BUFFER(reader io.ReaderAt, offset int64) *LVKEY_BUFFER { - return &LVKEY_BUFFER{Reader: reader, Offset: offset, Profile: self} + return &LVKEY_BUFFER{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) Misc(reader io.ReaderAt, offset int64) *Misc { - return &Misc{Reader: reader, Offset: offset, Profile: self} + return &Misc{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) PageHeader_(reader io.ReaderAt, offset int64) *PageHeader_ { - return &PageHeader_{Reader: reader, Offset: offset, Profile: self} + return &PageHeader_{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) RecordTag(reader io.ReaderAt, offset int64) *RecordTag { - return &RecordTag{Reader: reader, Offset: offset, Profile: self} + return &RecordTag{Reader: reader, Offset: offset, Profile: self} } func (self *ESEProfile) Tag(reader io.ReaderAt, offset int64) *Tag { - return &Tag{Reader: reader, Offset: offset, Profile: self} + return &Tag{Reader: reader, Offset: offset, Profile: self} } - type CATALOG_TYPE_COLUMN struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *CATALOG_TYPE_COLUMN) Size() int { - return 0 + return 0 } func (self *CATALOG_TYPE_COLUMN) ColumnType() *Enumeration { - value := ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_ColumnType + self.Offset) - name := "Unknown" - switch value { + value := ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_ColumnType+self.Offset) + name := "Unknown" + switch value { - case 0: - name = "NULL" + case 0: + name = "NULL" - case 1: - name = "Boolean" + case 1: + name = "Boolean" - case 2: - name = "Signed byte" + case 2: + name = "Signed byte" - case 3: - name = "Signed short" + case 3: + name = "Signed short" - case 4: - name = "Signed long" + case 4: + name = "Signed long" - case 5: - name = "Currency" + case 5: + name = "Currency" - case 6: - name = "Single precision FP" + case 6: + name = "Single precision FP" - case 7: - name = "Double precision FP" + case 7: + name = "Double precision FP" - case 8: - name = "DateTime" + case 8: + name = "DateTime" - case 9: - name = "Binary" + case 9: + name = "Binary" - case 10: - name = "Text" + case 10: + name = "Text" - case 11: - name = "Long Binary" + case 11: + name = "Long Binary" - case 12: - name = "Long Text" + case 12: + name = "Long Text" - case 13: - name = "Obsolete" + case 13: + name = "Obsolete" - case 14: - name = "Unsigned long" + case 14: + name = "Unsigned long" - case 15: - name = "Long long" + case 15: + name = "Long long" - case 16: - name = "GUID" + case 16: + name = "GUID" - case 17: - name = "Unsigned short" + case 17: + name = "Unsigned short" - case 18: - name = "Max" -} - return &Enumeration{Value: uint64(value), Name: name} + case 18: + name = "Max" + } + return &Enumeration{Value: uint64(value), Name: name} } - func (self *CATALOG_TYPE_COLUMN) SpaceUsage() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_SpaceUsage + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_SpaceUsage+self.Offset) } func (self *CATALOG_TYPE_COLUMN) ColumnFlags() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_ColumnFlags + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_ColumnFlags+self.Offset) } func (self *CATALOG_TYPE_COLUMN) CodePage() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_CodePage + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_CodePage+self.Offset) } func (self *CATALOG_TYPE_COLUMN) DebugString() string { - result := fmt.Sprintf("struct CATALOG_TYPE_COLUMN @ %#x:\n", self.Offset) - result += fmt.Sprintf(" ColumnType: %v\n", self.ColumnType().DebugString()) - result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) - result += fmt.Sprintf(" ColumnFlags: %#0x\n", self.ColumnFlags()) - result += fmt.Sprintf(" CodePage: %#0x\n", self.CodePage()) - return result + result := fmt.Sprintf("struct CATALOG_TYPE_COLUMN @ %#x:\n", self.Offset) + result += fmt.Sprintf(" ColumnType: %v\n", self.ColumnType().DebugString()) + result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) + result += fmt.Sprintf(" ColumnFlags: %#0x\n", self.ColumnFlags()) + result += fmt.Sprintf(" CodePage: %#0x\n", self.CodePage()) + return result } type CATALOG_TYPE_INDEX struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *CATALOG_TYPE_INDEX) Size() int { - return 0 + return 0 } func (self *CATALOG_TYPE_INDEX) FatherDataPageNumber() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_FatherDataPageNumber + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_FatherDataPageNumber+self.Offset) } func (self *CATALOG_TYPE_INDEX) SpaceUsage() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_SpaceUsage + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_SpaceUsage+self.Offset) } func (self *CATALOG_TYPE_INDEX) IndexFlags() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_IndexFlags + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_IndexFlags+self.Offset) } func (self *CATALOG_TYPE_INDEX) Locale() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_Locale + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_Locale+self.Offset) } func (self *CATALOG_TYPE_INDEX) DebugString() string { - result := fmt.Sprintf("struct CATALOG_TYPE_INDEX @ %#x:\n", self.Offset) - result += fmt.Sprintf(" FatherDataPageNumber: %#0x\n", self.FatherDataPageNumber()) - result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) - result += fmt.Sprintf(" IndexFlags: %#0x\n", self.IndexFlags()) - result += fmt.Sprintf(" Locale: %#0x\n", self.Locale()) - return result + result := fmt.Sprintf("struct CATALOG_TYPE_INDEX @ %#x:\n", self.Offset) + result += fmt.Sprintf(" FatherDataPageNumber: %#0x\n", self.FatherDataPageNumber()) + result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) + result += fmt.Sprintf(" IndexFlags: %#0x\n", self.IndexFlags()) + result += fmt.Sprintf(" Locale: %#0x\n", self.Locale()) + return result } type CATALOG_TYPE_LONG_VALUE struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *CATALOG_TYPE_LONG_VALUE) Size() int { - return 0 + return 0 } func (self *CATALOG_TYPE_LONG_VALUE) FatherDataPageNumber() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_FatherDataPageNumber + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_FatherDataPageNumber+self.Offset) } func (self *CATALOG_TYPE_LONG_VALUE) SpaceUsage() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_SpaceUsage + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_SpaceUsage+self.Offset) } func (self *CATALOG_TYPE_LONG_VALUE) LVFlags() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_LVFlags + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_LVFlags+self.Offset) } func (self *CATALOG_TYPE_LONG_VALUE) InitialNumberOfPages() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_InitialNumberOfPages + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_InitialNumberOfPages+self.Offset) } func (self *CATALOG_TYPE_LONG_VALUE) DebugString() string { - result := fmt.Sprintf("struct CATALOG_TYPE_LONG_VALUE @ %#x:\n", self.Offset) - result += fmt.Sprintf(" FatherDataPageNumber: %#0x\n", self.FatherDataPageNumber()) - result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) - result += fmt.Sprintf(" LVFlags: %#0x\n", self.LVFlags()) - result += fmt.Sprintf(" InitialNumberOfPages: %#0x\n", self.InitialNumberOfPages()) - return result + result := fmt.Sprintf("struct CATALOG_TYPE_LONG_VALUE @ %#x:\n", self.Offset) + result += fmt.Sprintf(" FatherDataPageNumber: %#0x\n", self.FatherDataPageNumber()) + result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) + result += fmt.Sprintf(" LVFlags: %#0x\n", self.LVFlags()) + result += fmt.Sprintf(" InitialNumberOfPages: %#0x\n", self.InitialNumberOfPages()) + return result } type CATALOG_TYPE_TABLE struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *CATALOG_TYPE_TABLE) Size() int { - return 0 + return 0 } func (self *CATALOG_TYPE_TABLE) FatherDataPageNumber() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_TABLE_FatherDataPageNumber + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_TABLE_FatherDataPageNumber+self.Offset) } func (self *CATALOG_TYPE_TABLE) SpaceUsage() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_TABLE_SpaceUsage + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_TABLE_SpaceUsage+self.Offset) } func (self *CATALOG_TYPE_TABLE) DebugString() string { - result := fmt.Sprintf("struct CATALOG_TYPE_TABLE @ %#x:\n", self.Offset) - result += fmt.Sprintf(" FatherDataPageNumber: %#0x\n", self.FatherDataPageNumber()) - result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) - return result + result := fmt.Sprintf("struct CATALOG_TYPE_TABLE @ %#x:\n", self.Offset) + result += fmt.Sprintf(" FatherDataPageNumber: %#0x\n", self.FatherDataPageNumber()) + result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) + return result } type DBTime struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *DBTime) Size() int { - return 8 + return 8 } func (self *DBTime) Hours() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_DBTime_Hours + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_DBTime_Hours+self.Offset) } func (self *DBTime) Min() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_DBTime_Min + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_DBTime_Min+self.Offset) } func (self *DBTime) Sec() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_DBTime_Sec + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_DBTime_Sec+self.Offset) } func (self *DBTime) DebugString() string { - result := fmt.Sprintf("struct DBTime @ %#x:\n", self.Offset) - result += fmt.Sprintf(" Hours: %#0x\n", self.Hours()) - result += fmt.Sprintf(" Min: %#0x\n", self.Min()) - result += fmt.Sprintf(" Sec: %#0x\n", self.Sec()) - return result + result := fmt.Sprintf("struct DBTime @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Hours: %#0x\n", self.Hours()) + result += fmt.Sprintf(" Min: %#0x\n", self.Min()) + result += fmt.Sprintf(" Sec: %#0x\n", self.Sec()) + return result } type ESENT_BRANCH_ENTRY struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *ESENT_BRANCH_ENTRY) Size() int { - return 16 + return 16 } func (self *ESENT_BRANCH_ENTRY) LocalPageKeySize() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_ESENT_BRANCH_ENTRY_LocalPageKeySize + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_ESENT_BRANCH_ENTRY_LocalPageKeySize+self.Offset) } func (self *ESENT_BRANCH_ENTRY) DebugString() string { - result := fmt.Sprintf("struct ESENT_BRANCH_ENTRY @ %#x:\n", self.Offset) - result += fmt.Sprintf(" LocalPageKeySize: %#0x\n", self.LocalPageKeySize()) - return result + result := fmt.Sprintf("struct ESENT_BRANCH_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" LocalPageKeySize: %#0x\n", self.LocalPageKeySize()) + return result } type ESENT_BRANCH_HEADER struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *ESENT_BRANCH_HEADER) Size() int { - return 16 + return 16 } - func (self *ESENT_BRANCH_HEADER) CommonPageKey() string { - return ParseTerminatedString(self.Reader, self.Profile.Off_ESENT_BRANCH_HEADER_CommonPageKey + self.Offset) + return ParseTerminatedString(self.Reader, self.Profile.Off_ESENT_BRANCH_HEADER_CommonPageKey+self.Offset) } func (self *ESENT_BRANCH_HEADER) DebugString() string { - result := fmt.Sprintf("struct ESENT_BRANCH_HEADER @ %#x:\n", self.Offset) - result += fmt.Sprintf(" CommonPageKey: %v\n", string(self.CommonPageKey())) - return result + result := fmt.Sprintf("struct ESENT_BRANCH_HEADER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" CommonPageKey: %v\n", string(self.CommonPageKey())) + return result } type ESENT_CATALOG_DATA_DEFINITION_ENTRY struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Size() int { - return 0 + return 0 } func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) FDPId() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_FDPId + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_FDPId+self.Offset) } func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Type() *Enumeration { - value := ParseUint16(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Type + self.Offset) - name := "Unknown" - switch value { + value := ParseUint16(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Type+self.Offset) + name := "Unknown" + switch value { - case 1: - name = "CATALOG_TYPE_TABLE" + case 1: + name = "CATALOG_TYPE_TABLE" - case 2: - name = "CATALOG_TYPE_COLUMN" + case 2: + name = "CATALOG_TYPE_COLUMN" - case 3: - name = "CATALOG_TYPE_INDEX" + case 3: + name = "CATALOG_TYPE_INDEX" - case 4: - name = "CATALOG_TYPE_LONG_VALUE" -} - return &Enumeration{Value: uint64(value), Name: name} + case 4: + name = "CATALOG_TYPE_LONG_VALUE" + } + return &Enumeration{Value: uint64(value), Name: name} } - func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Identifier() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Identifier + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Identifier+self.Offset) } func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Column() *CATALOG_TYPE_COLUMN { - return self.Profile.CATALOG_TYPE_COLUMN(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Column + self.Offset) + return self.Profile.CATALOG_TYPE_COLUMN(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Column+self.Offset) } func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Table() *CATALOG_TYPE_TABLE { - return self.Profile.CATALOG_TYPE_TABLE(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Table + self.Offset) + return self.Profile.CATALOG_TYPE_TABLE(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Table+self.Offset) } func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Index() *CATALOG_TYPE_INDEX { - return self.Profile.CATALOG_TYPE_INDEX(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Index + self.Offset) + return self.Profile.CATALOG_TYPE_INDEX(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Index+self.Offset) } func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) LongValue() *CATALOG_TYPE_LONG_VALUE { - return self.Profile.CATALOG_TYPE_LONG_VALUE(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_LongValue + self.Offset) + return self.Profile.CATALOG_TYPE_LONG_VALUE(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_LongValue+self.Offset) } func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) DebugString() string { - result := fmt.Sprintf("struct ESENT_CATALOG_DATA_DEFINITION_ENTRY @ %#x:\n", self.Offset) - result += fmt.Sprintf(" FDPId: %#0x\n", self.FDPId()) - result += fmt.Sprintf(" Type: %v\n", self.Type().DebugString()) - result += fmt.Sprintf(" Identifier: %#0x\n", self.Identifier()) - result += fmt.Sprintf(" Column: {\n%v}\n", indent(self.Column().DebugString())) - result += fmt.Sprintf(" Table: {\n%v}\n", indent(self.Table().DebugString())) - result += fmt.Sprintf(" Index: {\n%v}\n", indent(self.Index().DebugString())) - result += fmt.Sprintf(" LongValue: {\n%v}\n", indent(self.LongValue().DebugString())) - return result + result := fmt.Sprintf("struct ESENT_CATALOG_DATA_DEFINITION_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" FDPId: %#0x\n", self.FDPId()) + result += fmt.Sprintf(" Type: %v\n", self.Type().DebugString()) + result += fmt.Sprintf(" Identifier: %#0x\n", self.Identifier()) + result += fmt.Sprintf(" Column: {\n%v}\n", indent(self.Column().DebugString())) + result += fmt.Sprintf(" Table: {\n%v}\n", indent(self.Table().DebugString())) + result += fmt.Sprintf(" Index: {\n%v}\n", indent(self.Index().DebugString())) + result += fmt.Sprintf(" LongValue: {\n%v}\n", indent(self.LongValue().DebugString())) + return result } type ESENT_DATA_DEFINITION_HEADER struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *ESENT_DATA_DEFINITION_HEADER) Size() int { - return 4 + return 4 } func (self *ESENT_DATA_DEFINITION_HEADER) LastFixedType() byte { - return ParseUint8(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_LastFixedType + self.Offset) + return ParseUint8(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_LastFixedType+self.Offset) } func (self *ESENT_DATA_DEFINITION_HEADER) LastVariableDataType() byte { - return ParseUint8(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_LastVariableDataType + self.Offset) + return ParseUint8(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_LastVariableDataType+self.Offset) } func (self *ESENT_DATA_DEFINITION_HEADER) VariableSizeOffset() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_VariableSizeOffset + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_VariableSizeOffset+self.Offset) } func (self *ESENT_DATA_DEFINITION_HEADER) DebugString() string { - result := fmt.Sprintf("struct ESENT_DATA_DEFINITION_HEADER @ %#x:\n", self.Offset) - result += fmt.Sprintf(" LastFixedType: %#0x\n", self.LastFixedType()) - result += fmt.Sprintf(" LastVariableDataType: %#0x\n", self.LastVariableDataType()) - result += fmt.Sprintf(" VariableSizeOffset: %#0x\n", self.VariableSizeOffset()) - return result + result := fmt.Sprintf("struct ESENT_DATA_DEFINITION_HEADER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" LastFixedType: %#0x\n", self.LastFixedType()) + result += fmt.Sprintf(" LastVariableDataType: %#0x\n", self.LastVariableDataType()) + result += fmt.Sprintf(" VariableSizeOffset: %#0x\n", self.VariableSizeOffset()) + return result } type ESENT_INDEX_ENTRY struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *ESENT_INDEX_ENTRY) Size() int { - return 16 + return 16 } - func (self *ESENT_INDEX_ENTRY) RecordPageKey() string { - return ParseTerminatedString(self.Reader, self.Profile.Off_ESENT_INDEX_ENTRY_RecordPageKey + self.Offset) + return ParseTerminatedString(self.Reader, self.Profile.Off_ESENT_INDEX_ENTRY_RecordPageKey+self.Offset) } func (self *ESENT_INDEX_ENTRY) DebugString() string { - result := fmt.Sprintf("struct ESENT_INDEX_ENTRY @ %#x:\n", self.Offset) - result += fmt.Sprintf(" RecordPageKey: %v\n", string(self.RecordPageKey())) - return result + result := fmt.Sprintf("struct ESENT_INDEX_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" RecordPageKey: %v\n", string(self.RecordPageKey())) + return result } type ESENT_LEAF_ENTRY struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *ESENT_LEAF_ENTRY) Size() int { - return 16 + return 16 } func (self *ESENT_LEAF_ENTRY) CommonPageKeySize() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_ESENT_LEAF_ENTRY_CommonPageKeySize + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_ESENT_LEAF_ENTRY_CommonPageKeySize+self.Offset) } func (self *ESENT_LEAF_ENTRY) LocalPageKeySize() uint64 { - value := ParseUint16(self.Reader, self.Profile.Off_ESENT_LEAF_ENTRY_LocalPageKeySize + self.Offset) - return (uint64(value) & 0x1fff) >> 0x0 + value := ParseUint16(self.Reader, self.Profile.Off_ESENT_LEAF_ENTRY_LocalPageKeySize+self.Offset) + return (uint64(value) & 0x1fff) >> 0x0 } func (self *ESENT_LEAF_ENTRY) DebugString() string { - result := fmt.Sprintf("struct ESENT_LEAF_ENTRY @ %#x:\n", self.Offset) - result += fmt.Sprintf(" CommonPageKeySize: %#0x\n", self.CommonPageKeySize()) - result += fmt.Sprintf(" LocalPageKeySize: %#0x\n", self.LocalPageKeySize()) - return result + result := fmt.Sprintf("struct ESENT_LEAF_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" CommonPageKeySize: %#0x\n", self.CommonPageKeySize()) + result += fmt.Sprintf(" LocalPageKeySize: %#0x\n", self.LocalPageKeySize()) + return result } type ESENT_LEAF_HEADER struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *ESENT_LEAF_HEADER) Size() int { - return 16 + return 16 } - func (self *ESENT_LEAF_HEADER) CommonPageKey() string { - return ParseTerminatedString(self.Reader, self.Profile.Off_ESENT_LEAF_HEADER_CommonPageKey + self.Offset) + return ParseTerminatedString(self.Reader, self.Profile.Off_ESENT_LEAF_HEADER_CommonPageKey+self.Offset) } func (self *ESENT_LEAF_HEADER) DebugString() string { - result := fmt.Sprintf("struct ESENT_LEAF_HEADER @ %#x:\n", self.Offset) - result += fmt.Sprintf(" CommonPageKey: %v\n", string(self.CommonPageKey())) - return result + result := fmt.Sprintf("struct ESENT_LEAF_HEADER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" CommonPageKey: %v\n", string(self.CommonPageKey())) + return result } type ESENT_ROOT_HEADER struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *ESENT_ROOT_HEADER) Size() int { - return 16 + return 16 } func (self *ESENT_ROOT_HEADER) InitialNumberOfPages() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_InitialNumberOfPages + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_InitialNumberOfPages+self.Offset) } func (self *ESENT_ROOT_HEADER) ParentFDP() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_ParentFDP + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_ParentFDP+self.Offset) } func (self *ESENT_ROOT_HEADER) ExtentSpace() *Enumeration { - value := ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_ExtentSpace + self.Offset) - name := "Unknown" - switch value { + value := ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_ExtentSpace+self.Offset) + name := "Unknown" + switch value { - case 0: - name = "Single" + case 0: + name = "Single" - case 1: - name = "Multiple" -} - return &Enumeration{Value: uint64(value), Name: name} + case 1: + name = "Multiple" + } + return &Enumeration{Value: uint64(value), Name: name} } - func (self *ESENT_ROOT_HEADER) SpaceTreePageNumber() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_SpaceTreePageNumber + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_SpaceTreePageNumber+self.Offset) } func (self *ESENT_ROOT_HEADER) DebugString() string { - result := fmt.Sprintf("struct ESENT_ROOT_HEADER @ %#x:\n", self.Offset) - result += fmt.Sprintf(" InitialNumberOfPages: %#0x\n", self.InitialNumberOfPages()) - result += fmt.Sprintf(" ParentFDP: %#0x\n", self.ParentFDP()) - result += fmt.Sprintf(" ExtentSpace: %v\n", self.ExtentSpace().DebugString()) - result += fmt.Sprintf(" SpaceTreePageNumber: %#0x\n", self.SpaceTreePageNumber()) - return result + result := fmt.Sprintf("struct ESENT_ROOT_HEADER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" InitialNumberOfPages: %#0x\n", self.InitialNumberOfPages()) + result += fmt.Sprintf(" ParentFDP: %#0x\n", self.ParentFDP()) + result += fmt.Sprintf(" ExtentSpace: %v\n", self.ExtentSpace().DebugString()) + result += fmt.Sprintf(" SpaceTreePageNumber: %#0x\n", self.SpaceTreePageNumber()) + return result } type ESENT_SPACE_TREE_ENTRY struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *ESENT_SPACE_TREE_ENTRY) Size() int { - return 16 + return 16 } func (self *ESENT_SPACE_TREE_ENTRY) PageKeySize() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_ESENT_SPACE_TREE_ENTRY_PageKeySize + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_ESENT_SPACE_TREE_ENTRY_PageKeySize+self.Offset) } func (self *ESENT_SPACE_TREE_ENTRY) LastPageNumber() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_ESENT_SPACE_TREE_ENTRY_LastPageNumber + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_ESENT_SPACE_TREE_ENTRY_LastPageNumber+self.Offset) } func (self *ESENT_SPACE_TREE_ENTRY) NumberOfPages() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_ESENT_SPACE_TREE_ENTRY_NumberOfPages + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_ESENT_SPACE_TREE_ENTRY_NumberOfPages+self.Offset) } func (self *ESENT_SPACE_TREE_ENTRY) DebugString() string { - result := fmt.Sprintf("struct ESENT_SPACE_TREE_ENTRY @ %#x:\n", self.Offset) - result += fmt.Sprintf(" PageKeySize: %#0x\n", self.PageKeySize()) - result += fmt.Sprintf(" LastPageNumber: %#0x\n", self.LastPageNumber()) - result += fmt.Sprintf(" NumberOfPages: %#0x\n", self.NumberOfPages()) - return result + result := fmt.Sprintf("struct ESENT_SPACE_TREE_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" PageKeySize: %#0x\n", self.PageKeySize()) + result += fmt.Sprintf(" LastPageNumber: %#0x\n", self.LastPageNumber()) + result += fmt.Sprintf(" NumberOfPages: %#0x\n", self.NumberOfPages()) + return result } type ESENT_SPACE_TREE_HEADER struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *ESENT_SPACE_TREE_HEADER) Size() int { - return 16 + return 16 } func (self *ESENT_SPACE_TREE_HEADER) DebugString() string { - result := fmt.Sprintf("struct ESENT_SPACE_TREE_HEADER @ %#x:\n", self.Offset) - return result + result := fmt.Sprintf("struct ESENT_SPACE_TREE_HEADER @ %#x:\n", self.Offset) + return result } type FileHeader struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *FileHeader) Size() int { - return 0 + return 0 } func (self *FileHeader) Magic() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_FileHeader_Magic + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_FileHeader_Magic+self.Offset) } func (self *FileHeader) FormatVersion() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_FileHeader_FormatVersion + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_FileHeader_FormatVersion+self.Offset) } func (self *FileHeader) FormatRevision() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_FileHeader_FormatRevision + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_FileHeader_FormatRevision+self.Offset) } func (self *FileHeader) FileType() *Enumeration { - value := ParseUint32(self.Reader, self.Profile.Off_FileHeader_FileType + self.Offset) - name := "Unknown" - switch value { + value := ParseUint32(self.Reader, self.Profile.Off_FileHeader_FileType+self.Offset) + name := "Unknown" + switch value { - case 0: - name = "Database" + case 0: + name = "Database" - case 1: - name = "StreamingFile" -} - return &Enumeration{Value: uint64(value), Name: name} + case 1: + name = "StreamingFile" + } + return &Enumeration{Value: uint64(value), Name: name} } - func (self *FileHeader) DataBaseTime() *DBTime { - return self.Profile.DBTime(self.Reader, self.Profile.Off_FileHeader_DataBaseTime + self.Offset) + return self.Profile.DBTime(self.Reader, self.Profile.Off_FileHeader_DataBaseTime+self.Offset) } func (self *FileHeader) Signature() *JET_SIGNATURE { - return self.Profile.JET_SIGNATURE(self.Reader, self.Profile.Off_FileHeader_Signature + self.Offset) + return self.Profile.JET_SIGNATURE(self.Reader, self.Profile.Off_FileHeader_Signature+self.Offset) } func (self *FileHeader) PageSize() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_FileHeader_PageSize + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_FileHeader_PageSize+self.Offset) } func (self *FileHeader) DebugString() string { - result := fmt.Sprintf("struct FileHeader @ %#x:\n", self.Offset) - result += fmt.Sprintf(" Magic: %#0x\n", self.Magic()) - result += fmt.Sprintf(" FormatVersion: %#0x\n", self.FormatVersion()) - result += fmt.Sprintf(" FormatRevision: %#0x\n", self.FormatRevision()) - result += fmt.Sprintf(" FileType: %v\n", self.FileType().DebugString()) - result += fmt.Sprintf(" DataBaseTime: {\n%v}\n", indent(self.DataBaseTime().DebugString())) - result += fmt.Sprintf(" Signature: {\n%v}\n", indent(self.Signature().DebugString())) - result += fmt.Sprintf(" PageSize: %#0x\n", self.PageSize()) - return result + result := fmt.Sprintf("struct FileHeader @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Magic: %#0x\n", self.Magic()) + result += fmt.Sprintf(" FormatVersion: %#0x\n", self.FormatVersion()) + result += fmt.Sprintf(" FormatRevision: %#0x\n", self.FormatRevision()) + result += fmt.Sprintf(" FileType: %v\n", self.FileType().DebugString()) + result += fmt.Sprintf(" DataBaseTime: {\n%v}\n", indent(self.DataBaseTime().DebugString())) + result += fmt.Sprintf(" Signature: {\n%v}\n", indent(self.Signature().DebugString())) + result += fmt.Sprintf(" PageSize: %#0x\n", self.PageSize()) + return result } type GUID struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *GUID) Size() int { - return 16 + return 16 } func (self *GUID) Data1() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_GUID_Data1 + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_GUID_Data1+self.Offset) } func (self *GUID) Data2() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_GUID_Data2 + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_GUID_Data2+self.Offset) } func (self *GUID) Data3() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_GUID_Data3 + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_GUID_Data3+self.Offset) } func (self *GUID) Data4() []byte { - return ParseArray_byte(self.Profile, self.Reader, self.Profile.Off_GUID_Data4 + self.Offset, 8) + return ParseArray_byte(self.Profile, self.Reader, self.Profile.Off_GUID_Data4+self.Offset, 8) } func (self *GUID) DebugString() string { - result := fmt.Sprintf("struct GUID @ %#x:\n", self.Offset) - result += fmt.Sprintf(" Data1: %#0x\n", self.Data1()) - result += fmt.Sprintf(" Data2: %#0x\n", self.Data2()) - result += fmt.Sprintf(" Data3: %#0x\n", self.Data3()) - return result + result := fmt.Sprintf("struct GUID @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Data1: %#0x\n", self.Data1()) + result += fmt.Sprintf(" Data2: %#0x\n", self.Data2()) + result += fmt.Sprintf(" Data3: %#0x\n", self.Data3()) + return result } type JET_LOGTIME struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *JET_LOGTIME) Size() int { - return 8 + return 8 } func (self *JET_LOGTIME) Sec() byte { - return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Sec + self.Offset) + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Sec+self.Offset) } func (self *JET_LOGTIME) Min() byte { - return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Min + self.Offset) + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Min+self.Offset) } func (self *JET_LOGTIME) Hours() byte { - return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Hours + self.Offset) + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Hours+self.Offset) } func (self *JET_LOGTIME) Days() byte { - return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Days + self.Offset) + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Days+self.Offset) } func (self *JET_LOGTIME) Month() byte { - return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Month + self.Offset) + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Month+self.Offset) } func (self *JET_LOGTIME) Year() byte { - return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Year + self.Offset) + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Year+self.Offset) } func (self *JET_LOGTIME) DebugString() string { - result := fmt.Sprintf("struct JET_LOGTIME @ %#x:\n", self.Offset) - result += fmt.Sprintf(" Sec: %#0x\n", self.Sec()) - result += fmt.Sprintf(" Min: %#0x\n", self.Min()) - result += fmt.Sprintf(" Hours: %#0x\n", self.Hours()) - result += fmt.Sprintf(" Days: %#0x\n", self.Days()) - result += fmt.Sprintf(" Month: %#0x\n", self.Month()) - result += fmt.Sprintf(" Year: %#0x\n", self.Year()) - return result + result := fmt.Sprintf("struct JET_LOGTIME @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Sec: %#0x\n", self.Sec()) + result += fmt.Sprintf(" Min: %#0x\n", self.Min()) + result += fmt.Sprintf(" Hours: %#0x\n", self.Hours()) + result += fmt.Sprintf(" Days: %#0x\n", self.Days()) + result += fmt.Sprintf(" Month: %#0x\n", self.Month()) + result += fmt.Sprintf(" Year: %#0x\n", self.Year()) + return result } type JET_SIGNATURE struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *JET_SIGNATURE) Size() int { - return 28 + return 28 } func (self *JET_SIGNATURE) Creation() *JET_LOGTIME { - return self.Profile.JET_LOGTIME(self.Reader, self.Profile.Off_JET_SIGNATURE_Creation + self.Offset) + return self.Profile.JET_LOGTIME(self.Reader, self.Profile.Off_JET_SIGNATURE_Creation+self.Offset) } - func (self *JET_SIGNATURE) CreatorMachine() string { - return ParseTerminatedString(self.Reader, self.Profile.Off_JET_SIGNATURE_CreatorMachine + self.Offset) + return ParseTerminatedString(self.Reader, self.Profile.Off_JET_SIGNATURE_CreatorMachine+self.Offset) } func (self *JET_SIGNATURE) DebugString() string { - result := fmt.Sprintf("struct JET_SIGNATURE @ %#x:\n", self.Offset) - result += fmt.Sprintf(" Creation: {\n%v}\n", indent(self.Creation().DebugString())) - result += fmt.Sprintf(" CreatorMachine: %v\n", string(self.CreatorMachine())) - return result + result := fmt.Sprintf("struct JET_SIGNATURE @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Creation: {\n%v}\n", indent(self.Creation().DebugString())) + result += fmt.Sprintf(" CreatorMachine: %v\n", string(self.CreatorMachine())) + return result } type LVKEY32 struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *LVKEY32) Size() int { - return 8 + return 8 } func (self *LVKEY32) Lid() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_LVKEY32_Lid + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_LVKEY32_Lid+self.Offset) } func (self *LVKEY32) SegmentOffset() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_LVKEY32_SegmentOffset + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_LVKEY32_SegmentOffset+self.Offset) } func (self *LVKEY32) DebugString() string { - result := fmt.Sprintf("struct LVKEY32 @ %#x:\n", self.Offset) - result += fmt.Sprintf(" Lid: %#0x\n", self.Lid()) - result += fmt.Sprintf(" SegmentOffset: %#0x\n", self.SegmentOffset()) - return result + result := fmt.Sprintf("struct LVKEY32 @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Lid: %#0x\n", self.Lid()) + result += fmt.Sprintf(" SegmentOffset: %#0x\n", self.SegmentOffset()) + return result } type LVKEY64 struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *LVKEY64) Size() int { - return 12 + return 12 } func (self *LVKEY64) Lid() uint64 { - return ParseUint64(self.Reader, self.Profile.Off_LVKEY64_Lid + self.Offset) + return ParseUint64(self.Reader, self.Profile.Off_LVKEY64_Lid+self.Offset) } func (self *LVKEY64) SegmentOffset() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_LVKEY64_SegmentOffset + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_LVKEY64_SegmentOffset+self.Offset) } func (self *LVKEY64) DebugString() string { - result := fmt.Sprintf("struct LVKEY64 @ %#x:\n", self.Offset) - result += fmt.Sprintf(" Lid: %#0x\n", self.Lid()) - result += fmt.Sprintf(" SegmentOffset: %#0x\n", self.SegmentOffset()) - return result + result := fmt.Sprintf("struct LVKEY64 @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Lid: %#0x\n", self.Lid()) + result += fmt.Sprintf(" SegmentOffset: %#0x\n", self.SegmentOffset()) + return result } type LVKEY_BUFFER struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *LVKEY_BUFFER) Size() int { - return 0 + return 0 } func (self *LVKEY_BUFFER) PrefixLength() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_LVKEY_BUFFER_PrefixLength + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_LVKEY_BUFFER_PrefixLength+self.Offset) } func (self *LVKEY_BUFFER) SuffixLength() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_LVKEY_BUFFER_SuffixLength + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_LVKEY_BUFFER_SuffixLength+self.Offset) } - func (self *LVKEY_BUFFER) KeyBuffer() string { - return ParseString(self.Reader, self.Profile.Off_LVKEY_BUFFER_KeyBuffer + self.Offset, 12) + return ParseString(self.Reader, self.Profile.Off_LVKEY_BUFFER_KeyBuffer+self.Offset, 12) } func (self *LVKEY_BUFFER) DebugString() string { - result := fmt.Sprintf("struct LVKEY_BUFFER @ %#x:\n", self.Offset) - result += fmt.Sprintf(" PrefixLength: %#0x\n", self.PrefixLength()) - result += fmt.Sprintf(" SuffixLength: %#0x\n", self.SuffixLength()) - result += fmt.Sprintf(" KeyBuffer: %v\n", string(self.KeyBuffer())) - return result + result := fmt.Sprintf("struct LVKEY_BUFFER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" PrefixLength: %#0x\n", self.PrefixLength()) + result += fmt.Sprintf(" SuffixLength: %#0x\n", self.SuffixLength()) + result += fmt.Sprintf(" KeyBuffer: %v\n", string(self.KeyBuffer())) + return result } type Misc struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *Misc) Size() int { - return 0 + return 0 } func (self *Misc) Misc() int32 { - return ParseInt32(self.Reader, self.Profile.Off_Misc_Misc + self.Offset) + return ParseInt32(self.Reader, self.Profile.Off_Misc_Misc+self.Offset) } func (self *Misc) Misc2() int16 { - return ParseInt16(self.Reader, self.Profile.Off_Misc_Misc2 + self.Offset) + return ParseInt16(self.Reader, self.Profile.Off_Misc_Misc2+self.Offset) } func (self *Misc) Misc3() int64 { - return ParseInt64(self.Reader, self.Profile.Off_Misc_Misc3 + self.Offset) + return ParseInt64(self.Reader, self.Profile.Off_Misc_Misc3+self.Offset) } func (self *Misc) Misc5() uint64 { - return ParseUint64(self.Reader, self.Profile.Off_Misc_Misc5 + self.Offset) + return ParseUint64(self.Reader, self.Profile.Off_Misc_Misc5+self.Offset) } - func (self *Misc) Misc4() string { - return ParseTerminatedUTF16String(self.Reader, self.Profile.Off_Misc_Misc4 + self.Offset) + return ParseTerminatedUTF16String(self.Reader, self.Profile.Off_Misc_Misc4+self.Offset) } func (self *Misc) DebugString() string { - result := fmt.Sprintf("struct Misc @ %#x:\n", self.Offset) - result += fmt.Sprintf(" Misc: %#0x\n", self.Misc()) - result += fmt.Sprintf(" Misc2: %#0x\n", self.Misc2()) - result += fmt.Sprintf(" Misc3: %#0x\n", self.Misc3()) - result += fmt.Sprintf(" Misc5: %#0x\n", self.Misc5()) - result += fmt.Sprintf(" Misc4: %v\n", string(self.Misc4())) - return result + result := fmt.Sprintf("struct Misc @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Misc: %#0x\n", self.Misc()) + result += fmt.Sprintf(" Misc2: %#0x\n", self.Misc2()) + result += fmt.Sprintf(" Misc3: %#0x\n", self.Misc3()) + result += fmt.Sprintf(" Misc5: %#0x\n", self.Misc5()) + result += fmt.Sprintf(" Misc4: %v\n", string(self.Misc4())) + return result } type PageHeader_ struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *PageHeader_) Size() int { - return 0 + return 0 } func (self *PageHeader_) LastModified() *DBTime { - return self.Profile.DBTime(self.Reader, self.Profile.Off_PageHeader__LastModified + self.Offset) + return self.Profile.DBTime(self.Reader, self.Profile.Off_PageHeader__LastModified+self.Offset) } func (self *PageHeader_) PreviousPageNumber() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_PageHeader__PreviousPageNumber + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_PageHeader__PreviousPageNumber+self.Offset) } func (self *PageHeader_) NextPageNumber() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_PageHeader__NextPageNumber + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_PageHeader__NextPageNumber+self.Offset) } func (self *PageHeader_) FatherPage() uint32 { - return ParseUint32(self.Reader, self.Profile.Off_PageHeader__FatherPage + self.Offset) + return ParseUint32(self.Reader, self.Profile.Off_PageHeader__FatherPage+self.Offset) } func (self *PageHeader_) AvailableDataSize() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_PageHeader__AvailableDataSize + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_PageHeader__AvailableDataSize+self.Offset) } func (self *PageHeader_) AvailableDataOffset() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_PageHeader__AvailableDataOffset + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_PageHeader__AvailableDataOffset+self.Offset) } func (self *PageHeader_) AvailablePageTag() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_PageHeader__AvailablePageTag + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_PageHeader__AvailablePageTag+self.Offset) } func (self *PageHeader_) Flags() *Flags { - value := ParseUint32(self.Reader, self.Profile.Off_PageHeader__Flags + self.Offset) - names := make(map[string]bool) + value := ParseUint32(self.Reader, self.Profile.Off_PageHeader__Flags+self.Offset) + names := make(map[string]bool) + if value&1 != 0 { + names["Root"] = true + } - if value & 1 != 0 { - names["Root"] = true - } - - if value & 2 != 0 { - names["Leaf"] = true - } + if value&2 != 0 { + names["Leaf"] = true + } - if value & 4 != 0 { - names["Parent"] = true - } + if value&4 != 0 { + names["Parent"] = true + } - if value & 8 != 0 { - names["Empty"] = true - } + if value&8 != 0 { + names["Empty"] = true + } - if value & 32 != 0 { - names["SpaceTree"] = true - } + if value&32 != 0 { + names["SpaceTree"] = true + } - if value & 64 != 0 { - names["Index"] = true - } + if value&64 != 0 { + names["Index"] = true + } - if value & 128 != 0 { - names["Long"] = true - } + if value&128 != 0 { + names["Long"] = true + } - return &Flags{Value: uint64(value), Names: names} + return &Flags{Value: uint64(value), Names: names} } func (self *PageHeader_) DebugString() string { - result := fmt.Sprintf("struct PageHeader_ @ %#x:\n", self.Offset) - result += fmt.Sprintf(" LastModified: {\n%v}\n", indent(self.LastModified().DebugString())) - result += fmt.Sprintf(" PreviousPageNumber: %#0x\n", self.PreviousPageNumber()) - result += fmt.Sprintf(" NextPageNumber: %#0x\n", self.NextPageNumber()) - result += fmt.Sprintf(" FatherPage: %#0x\n", self.FatherPage()) - result += fmt.Sprintf(" AvailableDataSize: %#0x\n", self.AvailableDataSize()) - result += fmt.Sprintf(" AvailableDataOffset: %#0x\n", self.AvailableDataOffset()) - result += fmt.Sprintf(" AvailablePageTag: %#0x\n", self.AvailablePageTag()) - result += fmt.Sprintf(" Flags: %v\n", self.Flags().DebugString()) - return result + result := fmt.Sprintf("struct PageHeader_ @ %#x:\n", self.Offset) + result += fmt.Sprintf(" LastModified: {\n%v}\n", indent(self.LastModified().DebugString())) + result += fmt.Sprintf(" PreviousPageNumber: %#0x\n", self.PreviousPageNumber()) + result += fmt.Sprintf(" NextPageNumber: %#0x\n", self.NextPageNumber()) + result += fmt.Sprintf(" FatherPage: %#0x\n", self.FatherPage()) + result += fmt.Sprintf(" AvailableDataSize: %#0x\n", self.AvailableDataSize()) + result += fmt.Sprintf(" AvailableDataOffset: %#0x\n", self.AvailableDataOffset()) + result += fmt.Sprintf(" AvailablePageTag: %#0x\n", self.AvailablePageTag()) + result += fmt.Sprintf(" Flags: %v\n", self.Flags().DebugString()) + return result } type RecordTag struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *RecordTag) Size() int { - return 4 + return 4 } func (self *RecordTag) Identifier() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_RecordTag_Identifier + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_RecordTag_Identifier+self.Offset) } func (self *RecordTag) DataOffset() uint64 { - value := ParseUint16(self.Reader, self.Profile.Off_RecordTag_DataOffset + self.Offset) - return (uint64(value) & 0x1fff) >> 0x0 + value := ParseUint16(self.Reader, self.Profile.Off_RecordTag_DataOffset+self.Offset) + return (uint64(value) & 0x1fff) >> 0x0 } func (self *RecordTag) Flags() uint64 { - value := ParseUint16(self.Reader, self.Profile.Off_RecordTag_Flags + self.Offset) - return (uint64(value) & 0xffff) >> 0xe + value := ParseUint16(self.Reader, self.Profile.Off_RecordTag_Flags+self.Offset) + return (uint64(value) & 0xffff) >> 0xe } func (self *RecordTag) DebugString() string { - result := fmt.Sprintf("struct RecordTag @ %#x:\n", self.Offset) - result += fmt.Sprintf(" Identifier: %#0x\n", self.Identifier()) - result += fmt.Sprintf(" DataOffset: %#0x\n", self.DataOffset()) - result += fmt.Sprintf(" Flags: %#0x\n", self.Flags()) - return result + result := fmt.Sprintf("struct RecordTag @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Identifier: %#0x\n", self.Identifier()) + result += fmt.Sprintf(" DataOffset: %#0x\n", self.DataOffset()) + result += fmt.Sprintf(" Flags: %#0x\n", self.Flags()) + return result } type Tag struct { - Reader io.ReaderAt - Offset int64 - Profile *ESEProfile + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile } func (self *Tag) Size() int { - return 4 + return 4 } func (self *Tag) _ValueSize() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_Tag__ValueSize + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_Tag__ValueSize+self.Offset) } func (self *Tag) _ValueOffset() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_Tag__ValueOffset + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_Tag__ValueOffset+self.Offset) } func (self *Tag) Flags_() uint16 { - return ParseUint16(self.Reader, self.Profile.Off_Tag_Flags_ + self.Offset) + return ParseUint16(self.Reader, self.Profile.Off_Tag_Flags_+self.Offset) } func (self *Tag) Flags() *Flags { - value := ParseUint16(self.Reader, self.Profile.Off_Tag_Flags + self.Offset) - names := make(map[string]bool) + value := ParseUint16(self.Reader, self.Profile.Off_Tag_Flags+self.Offset) + names := make(map[string]bool) + if value&8192 != 0 { + names["fNDVersion"] = true + } - if value & 8192 != 0 { - names["fNDVersion"] = true - } - - if value & 16384 != 0 { - names["fNDDeleted"] = true - } + if value&16384 != 0 { + names["fNDDeleted"] = true + } - if value & 32768 != 0 { - names["fNDCompressed"] = true - } + if value&32768 != 0 { + names["fNDCompressed"] = true + } - return &Flags{Value: uint64(value), Names: names} + return &Flags{Value: uint64(value), Names: names} } func (self *Tag) DebugString() string { - result := fmt.Sprintf("struct Tag @ %#x:\n", self.Offset) - result += fmt.Sprintf(" _ValueSize: %#0x\n", self._ValueSize()) - result += fmt.Sprintf(" _ValueOffset: %#0x\n", self._ValueOffset()) - result += fmt.Sprintf(" Flags_: %#0x\n", self.Flags_()) - result += fmt.Sprintf(" Flags: %v\n", self.Flags().DebugString()) - return result + result := fmt.Sprintf("struct Tag @ %#x:\n", self.Offset) + result += fmt.Sprintf(" _ValueSize: %#0x\n", self._ValueSize()) + result += fmt.Sprintf(" _ValueOffset: %#0x\n", self._ValueOffset()) + result += fmt.Sprintf(" Flags_: %#0x\n", self.Flags_()) + result += fmt.Sprintf(" Flags: %v\n", self.Flags().DebugString()) + return result } type Enumeration struct { - Value uint64 - Name string + Value uint64 + Name string } func (self Enumeration) DebugString() string { - return fmt.Sprintf("%s (%d)", self.Name, self.Value) + return fmt.Sprintf("%s (%d)", self.Name, self.Value) } - type Flags struct { - Value uint64 - Names map[string]bool + Value uint64 + Names map[string]bool } func (self Flags) DebugString() string { - names := []string{} - for k, _ := range self.Names { - names = append(names, k) - } + names := []string{} + for k, _ := range self.Names { + names = append(names, k) + } - sort.Strings(names) + sort.Strings(names) - return fmt.Sprintf("%d (%s)", self.Value, strings.Join(names, ",")) + return fmt.Sprintf("%d (%s)", self.Value, strings.Join(names, ",")) } func (self Flags) IsSet(flag string) bool { - result, _ := self.Names[flag] - return result + result, _ := self.Names[flag] + return result } func (self Flags) Values() []string { - result := make([]string, 0, len(self.Names)) - for k, _ := range self.Names { - result = append(result, k) - } - return result + result := make([]string, 0, len(self.Names)) + for k, _ := range self.Names { + result = append(result, k) + } + return result } - func ParseArray_byte(profile *ESEProfile, reader io.ReaderAt, offset int64, count int) []byte { - result := make([]byte, 0, count) - for i:=0; i= 20 && t[10] == 'T' { - // Attempt to convert it from timestamp. - parsed, err := time.Parse(time.RFC3339, t) - if err == nil { - return parsed, nil - } - } - - return t, nil - - case json.Number: - value_str := t.String() - - // Try to parse as Uint - value_uint, err := strconv.ParseUint(value_str, 10, 64) - if err == nil { - return value_uint, nil - } - - value_int, err := strconv.ParseInt(value_str, 10, 64) - if err == nil { - return value_int, nil - } - - // Failing this, try a float - float, err := strconv.ParseFloat(value_str, 64) - if err == nil { - return float, nil - } - - return nil, fmt.Errorf("Unexpected token: %v", token) - } - return token, nil -} - -// Preserve key order when marshalling to JSON. -func (self *Dict) MarshalJSON() ([]byte, error) { - self.Lock() - defer self.Unlock() - - buf := &bytes.Buffer{} - buf.Write([]byte("{")) - for _, k := range self.keys { - - // add key - kEscaped, err := json.Marshal(k) - if err != nil { - continue - } - - // add value - v := self.store[k] - - // Check for back references and skip them - this is not perfect. - subdict, ok := v.(*Dict) - if ok && subdict == self { - continue - } - - buf.Write(kEscaped) - buf.Write([]byte(":")) - - vBytes, err := json.Marshal(v) - if err == nil { - buf.Write(vBytes) - buf.Write([]byte(",")) - } else { - buf.Write([]byte("null,")) - } - } - if len(self.keys) > 0 { - buf.Truncate(buf.Len() - 1) - } - buf.Write([]byte("}")) - return buf.Bytes(), nil -} - -func (self *Dict) MarshalYAML() (interface{}, error) { - self.Lock() - defer self.Unlock() - - result := yaml.MapSlice{} - for _, k := range self.keys { - v := self.store[k] - result = append(result, yaml.MapItem{ - Key: k, Value: v, - }) - } - - return result, nil -} diff --git a/internal/eseparser/ordereddict/ordereddict_test.go b/internal/eseparser/ordereddict/ordereddict_test.go deleted file mode 100644 index e2359ee..0000000 --- a/internal/eseparser/ordereddict/ordereddict_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package ordereddict - -import ( - "encoding/json" - "reflect" - "testing" - "time" - - "github.com/alecthomas/repr" - "github.com/stretchr/testify/assert" -) - -type dictSerializationTest struct { - dict *Dict - serialized string -} - -var ( - dictSerializationTests = []dictSerializationTest{ - {NewDict().Set("Foo", "Bar"), `{"Foo":"Bar"}`}, - - // Test an unserilizable member - This should not prevent the - // entire dict from serializing - only that member should be - // ignored. - {NewDict().Set("Foo", "Bar"). - Set("Time", time.Unix(3000000000000000, 0)), - `{"Foo":"Bar","Time":null}`}, - - // Recursive dict - {NewDict().Set("Foo", - NewDict().Set("Bar", 2). - Set("Time", time.Unix(3000000000000000, 0))), - `{"Foo":{"Bar":2,"Time":null}}`}, - - // Ensure key order is preserved. - {NewDict().Set("A", 1).Set("B", 2), `{"A":1,"B":2}`}, - {NewDict().Set("B", 1).Set("A", 2), `{"B":1,"A":2}`}, - - // Serialize with quotes - {NewDict().Set("foo\\'s quote", 1), `{"foo\\'s quote":1}`}, - } - - // Check that serialization decodes to the object. - dictUnserializationTest = []dictSerializationTest{ - // Preserve order of keys on deserialization. - {NewDict().Set("A", uint64(1)).Set("B", uint64(2)), `{"A":1,"B":2}`}, - {NewDict().Set("B", uint64(1)).Set("A", uint64(2)), `{"B":1,"A":2}`}, - - // Handle arrays, ints floats and bools - {NewDict().Set("B", uint64(1)).Set("A", uint64(2)), `{"B":1,"A":2}`}, - {NewDict().Set("B", float64(1)).Set("A", uint64(2)), `{"B":1.0,"A":2}`}, - {NewDict().Set("B", []interface{}{uint64(1)}).Set("A", uint64(2)), `{"B":[1],"A":2}`}, - {NewDict().Set("B", true).Set("A", uint64(2)), `{"B":true,"A":2}`}, - {NewDict().Set("B", nil).Set("A", uint64(2)), `{"B":null,"A":2}`}, - - // Embedded dicts decode into ordered dicts. - {NewDict(). - Set("B", NewDict().Set("Zoo", "X").Set("Baz", "Y")). - Set("A", "Z"), `{"B":{"Zoo":"X","Baz":"Y"},"A":"Z"}`}, - - // Make sure we properly preserve uint64 (overflows int64) - {NewDict(). - Set("Uint64", uint64(9223372036854775808)), - `{"Uint64": 9223372036854775808}`}, - - // We prefer uint64 but int64 is needed for negative numbers - {NewDict(). - Set("Int64", int64(-500)), - `{"Int64": -500}`}, - } -) - -func TestDictSerialization(t *testing.T) { - for _, test := range dictSerializationTests { - serialized, err := json.Marshal(test.dict) - if err != nil { - t.Fatalf("Failed to serialize %v: %v", repr.String(test.dict), err) - } - - assert.Equal(t, test.serialized, string(serialized)) - } -} - -func TestDictDeserialization(t *testing.T) { - for _, test := range dictUnserializationTest { - value := NewDict() - - err := json.Unmarshal([]byte(test.serialized), value) - if err != nil { - t.Fatalf("Failed to serialize %v: %v", test.serialized, err) - } - // Make sure the keys are the same. - assert.Equal(t, test.dict.Keys(), value.Keys()) - assert.Equal(t, test.dict.ToDict(), value.ToDict()) - - assert.True(t, reflect.DeepEqual(test.dict, value)) - } -} - -func TestTimestampDeserialization(t *testing.T) { - serialized := `{"Time":"2022-08-21T03:08:45.076884Z", "Time2":"2022-08-21T03:08:45.076884+07:00"}` - value := NewDict() - err := json.Unmarshal([]byte(serialized), value) - assert.NoError(t, err) - - t1_any, pres := value.Get("Time") - assert.True(t, pres) - - t1, ok := t1_any.(time.Time) - assert.True(t, ok) - assert.Equal(t, int64(1661051325), t1.Unix()) - - t2_any, pres := value.Get("Time2") - assert.True(t, pres) - - t2, ok := t2_any.(time.Time) - assert.True(t, ok) - assert.Equal(t, int64(1661026125), t2.Unix()) - -} - -func TestOrder(t *testing.T) { - test := NewDict(). - Set("A", 1). - Set("B", 2) - - assert.Equal(t, []string{"A", "B"}, test.Keys()) - - test = NewDict(). - Set("B", 1). - Set("A", 2) - - assert.Equal(t, []string{"B", "A"}, test.Keys()) -} - -func TestCaseInsensitive(t *testing.T) { - test := NewDict().SetCaseInsensitive() - - test.Set("FOO", 1) - - value, pres := test.Get("foo") - assert.True(t, pres) - assert.Equal(t, 1, value) - - test = NewDict().Set("FOO", 1) - value, pres = test.Get("foo") - assert.False(t, pres) -} diff --git a/internal/eseparser/pages.go b/internal/eseparser/pages.go index 5796484..8148118 100644 --- a/internal/eseparser/pages.go +++ b/internal/eseparser/pages.go @@ -1,4 +1,4 @@ -package parser +package eseparser import ( "fmt" diff --git a/internal/eseparser/reader.go b/internal/eseparser/reader.go index 4e18ad0..477edc0 100644 --- a/internal/eseparser/reader.go +++ b/internal/eseparser/reader.go @@ -1,4 +1,4 @@ -package parser +package eseparser import "io" diff --git a/internal/eseparser/utils.go b/internal/eseparser/utils.go index 5d820c7..b34101f 100644 --- a/internal/eseparser/utils.go +++ b/internal/eseparser/utils.go @@ -1,4 +1,4 @@ -package parser +package eseparser import ( "io" diff --git a/internal/ie/cookiestore.go b/internal/ie/cookiestore.go index 3faa0d4..5b6caad 100644 --- a/internal/ie/cookiestore.go +++ b/internal/ie/cookiestore.go @@ -7,9 +7,8 @@ import ( "github.com/browserutils/kooky" "github.com/browserutils/kooky/internal/cookies" + "github.com/browserutils/kooky/internal/eseparser" "github.com/browserutils/kooky/internal/utils" - - "www.velocidex.com/golang/go-ese/parser" ) type CookieStore struct { @@ -44,7 +43,7 @@ var _ cookies.CookieStore = (*IECacheCookieStore)(nil) type ESECookieStore struct { cookies.DefaultCookieStore - ESECatalog *parser.Catalog + ESECatalog *eseparser.Catalog } var _ cookies.CookieStore = (*ESECookieStore)(nil) @@ -71,12 +70,12 @@ func (s *ESECookieStore) Open() error { return err } - ese_ctx, err := parser.NewESEContext(s.File) + ese_ctx, err := eseparser.NewESEContext(s.File) if err != nil { return err } - catalog, err := parser.ReadCatalog(ese_ctx) + catalog, err := eseparser.ReadCatalog(ese_ctx) if err != nil { return err } diff --git a/internal/ie/ese.go b/internal/ie/ese.go index 51a8bf2..d249f03 100644 --- a/internal/ie/ese.go +++ b/internal/ie/ese.go @@ -9,8 +9,8 @@ import ( "github.com/browserutils/kooky" "github.com/browserutils/kooky/internal/timex" - "github.com/Velocidex/ordereddict" - "www.velocidex.com/golang/go-ese/parser" + "github.com/browserutils/kooky/internal/eseparser" + "github.com/browserutils/kooky/internal/eseparser/ordereddict" ) func (s *ESECookieStore) ReadCookies(filters ...kooky.Filter) ([]*kooky.Cookie, error) { @@ -212,13 +212,13 @@ type webCacheContainer struct { directory string // Directory } -func getEdgeCookieDirectories(catalog *parser.Catalog) ([]webCacheContainer, error) { +func getEdgeCookieDirectories(catalog *eseparser.Catalog) ([]webCacheContainer, error) { var cookiesContainers []webCacheContainer cbContainers := func(row *ordereddict.Dict) error { var name, directory string if n, ok := row.GetString(`Name`); ok { - name = strings.TrimRight(parser.UTF16BytesToUTF8([]byte(n), binary.LittleEndian), "\x00") + name = strings.TrimRight(eseparser.UTF16BytesToUTF8([]byte(n), binary.LittleEndian), "\x00") } else { return nil } From 910820c529b4a7e511b56ef6c9443a51f24127d5 Mon Sep 17 00:00:00 2001 From: Simon Lehn <48837958+srlehn@users.noreply.github.com> Date: Sat, 1 Jun 2024 00:41:51 +0200 Subject: [PATCH 40/40] remove CI test (ordereddict) --- .../ordereddict/.github/workflows/test.yaml | 20 ------------------- 1 file changed, 20 deletions(-) delete mode 100644 internal/eseparser/ordereddict/.github/workflows/test.yaml diff --git a/internal/eseparser/ordereddict/.github/workflows/test.yaml b/internal/eseparser/ordereddict/.github/workflows/test.yaml deleted file mode 100644 index 59e0dcd..0000000 --- a/internal/eseparser/ordereddict/.github/workflows/test.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: Test -on: [pull_request] -jobs: - build: - name: Windows Test - runs-on: ubuntu-latest - steps: - - name: Set up Go 1.20 - uses: actions/setup-go@v3 - with: - go-version: '^1.20' - id: go - - name: Check out code into the Go module directory - uses: actions/checkout@v3 - - - name: Test - shell: bash - if: always() - run: | - go test -v ./...