Sitelet https://github.com/Fantom-foundation/go-opera/commit/e03f7a2879f42f14fa79d4e78cbe4b06e90b11b6
Skip to content

Commit e03f7a2

Browse files
uprendisuprendis
committed
rework genesis file format
- genesis is self-hashed, making it secure to read from non-trusted source - genesis content uses .zip format and doesn't require O(n) pre-decoding - user may choose different genesis files for different purpose (e.g. archive, snapsync-only with blocks history, snapsync-only with no history)
1 parent 48a1e7c commit e03f7a2

51 files changed

Lines changed: 1065 additions & 1097 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cmd/opera/launcher/config.go‎

Lines changed: 93 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@ package launcher
22

33
import (
44
"bufio"
5-
"bytes"
65
"errors"
76
"fmt"
7+
"io"
88
"os"
99
"path"
1010
"path/filepath"
@@ -28,9 +28,11 @@ import (
2828
"github.com/Fantom-foundation/go-opera/gossip/emitter"
2929
"github.com/Fantom-foundation/go-opera/gossip/gasprice"
3030
"github.com/Fantom-foundation/go-opera/integration"
31-
"github.com/Fantom-foundation/go-opera/integration/makegenesis"
31+
"github.com/Fantom-foundation/go-opera/integration/makefakegenesis"
3232
"github.com/Fantom-foundation/go-opera/opera"
33+
"github.com/Fantom-foundation/go-opera/opera/genesis"
3334
"github.com/Fantom-foundation/go-opera/opera/genesisstore"
35+
"github.com/Fantom-foundation/go-opera/opera/genesisstore/fileszip"
3436
futils "github.com/Fantom-foundation/go-opera/utils"
3537
"github.com/Fantom-foundation/go-opera/vecmt"
3638
)
@@ -73,9 +75,13 @@ var (
7375
Value: DefaultCacheSize,
7476
}
7577
// GenesisFlag specifies network genesis configuration
76-
GenesisFlag = cli.StringFlag{
78+
GenesisFlag = cli.StringSliceFlag{
7779
Name: "genesis",
78-
Usage: "'path to genesis file' - sets the network genesis configuration.",
80+
Usage: "'path to genesis file(s)' - sets the network genesis configuration.",
81+
}
82+
ExperimentalGenesisFlag = cli.BoolFlag{
83+
Name: "genesis.allowExperimental",
84+
Usage: "Allow to use experimental genesis file.",
7985
}
8086

8187
RPCGlobalGasCapFlag = cli.Uint64Flag{
@@ -95,12 +101,40 @@ var (
95101
Value: "full",
96102
}
97103

98-
AllowedOperaGenesisHashes = map[uint64]hash.Hash{
99-
opera.MainNetworkID: hash.HexToHash("0x4a53c5445584b3bfc20dbfb2ec18ae20037c716f3ba2d9e1da768a9deca17cb4"),
100-
opera.TestNetworkID: hash.HexToHash("0xc4a5fc96e575a16a9a0c7349d44dc4d0f602a54e0a8543360c2fee4c3937b49e"),
104+
AllowedOperaGenesis = []GenesisTemplate{
105+
{
106+
Header: genesis.Header{
107+
GenesisID: hash.HexToHash("0x4a53c5445584b3bfc20dbfb2ec18ae20037c716f3ba2d9e1da768a9deca17cb4"),
108+
NetworkID: opera.MainNetworkID,
109+
NetworkName: "main",
110+
},
111+
Hashes: genesis.Hashes{
112+
Blocks: hash.Hashes{hash.HexToHash("0x4a53c5445584b3bfc20dbfb2ec18ae20037c716f3ba2d9e1da768a9deca17cb4")},
113+
Epochs: hash.Hashes{hash.HexToHash("0x4a53c5445584b3bfc20dbfb2ec18ae20037c716f3ba2d9e1da768a9deca17cb4")},
114+
RawEvmItems: hash.Hashes{hash.HexToHash("0x4a53c5445584b3bfc20dbfb2ec18ae20037c716f3ba2d9e1da768a9deca17cb4")},
115+
},
116+
},
117+
{
118+
Header: genesis.Header{
119+
GenesisID: hash.HexToHash("0xc4a5fc96e575a16a9a0c7349d44dc4d0f602a54e0a8543360c2fee4c3937b49e"),
120+
NetworkID: opera.TestNetworkID,
121+
NetworkName: "test",
122+
},
123+
Hashes: genesis.Hashes{
124+
Blocks: hash.Hashes{hash.HexToHash("0x4a53c5445584b3bfc20dbfb2ec18ae20037c716f3ba2d9e1da768a9deca17cb4")},
125+
Epochs: hash.Hashes{hash.HexToHash("0x4a53c5445584b3bfc20dbfb2ec18ae20037c716f3ba2d9e1da768a9deca17cb4")},
126+
RawEvmItems: hash.Hashes{hash.HexToHash("0x4a53c5445584b3bfc20dbfb2ec18ae20037c716f3ba2d9e1da768a9deca17cb4")},
127+
},
128+
},
101129
}
102130
)
103131

132+
type GenesisTemplate struct {
133+
Name string
134+
Header genesis.Header
135+
Hashes genesis.Hashes
136+
}
137+
104138
const (
105139
// DefaultCacheSize is calculated as memory consumption in a worst case scenario with default configuration
106140
// Average memory consumption might be 3-5 times lower than the maximum
@@ -135,12 +169,11 @@ type config struct {
135169

136170
func (c *config) AppConfigs() integration.Configs {
137171
return integration.Configs{
138-
Opera: c.Opera,
139-
OperaStore: c.OperaStore,
140-
Lachesis: c.Lachesis,
141-
LachesisStore: c.LachesisStore,
142-
VectorClock: c.VectorClock,
143-
AllowedGenesis: AllowedOperaGenesisHashes,
172+
Opera: c.Opera,
173+
OperaStore: c.OperaStore,
174+
Lachesis: c.Lachesis,
175+
LachesisStore: c.LachesisStore,
176+
VectorClock: c.VectorClock,
144177
}
145178
}
146179

@@ -164,51 +197,70 @@ func loadAllConfigs(file string, cfg *config) error {
164197
return err
165198
}
166199

167-
func getOperaGenesis(ctx *cli.Context) integration.InputGenesis {
168-
169-
var genesis integration.InputGenesis
200+
func getGenesisStore(ctx *cli.Context) *genesisstore.Store {
170201
switch {
171202
case ctx.GlobalIsSet(FakeNetFlag.Name):
172203
_, num, err := parseFakeGen(ctx.GlobalString(FakeNetFlag.Name))
173204
if err != nil {
174205
log.Crit("Invalid flag", "flag", FakeNetFlag.Name, "err", err)
175206
}
176-
fakeGenesisStore := makegenesis.FakeGenesisStore(2, num, futils.ToFtm(1000000000), futils.ToFtm(5000000))
177-
genesis = integration.InputGenesis{
178-
Hash: fakeGenesisStore.Hash(),
179-
Read: func(store *genesisstore.Store) error {
180-
buf := bytes.NewBuffer(nil)
181-
err = fakeGenesisStore.Export(buf)
182-
if err != nil {
183-
return err
184-
}
185-
return store.Import(buf)
186-
},
187-
Close: func() error {
188-
return nil
189-
},
190-
}
207+
return makefakegenesis.FakeGenesisStore(num, futils.ToFtm(1000000000), futils.ToFtm(5000000))
191208
case ctx.GlobalIsSet(GenesisFlag.Name):
192-
genesisPath := ctx.GlobalString(GenesisFlag.Name)
209+
genesisPaths := ctx.GlobalStringSlice(GenesisFlag.Name)
210+
var files []fileszip.Reader
211+
var closers []io.Closer
193212

194-
genesisFile, err := os.Open(genesisPath)
195-
if err != nil {
196-
utils.Fatalf("Failed to open genesis file: %v", err)
213+
for _, gPath := range genesisPaths {
214+
f, err := os.Open(gPath)
215+
if err != nil {
216+
utils.Fatalf("Failed to open genesis file: %v", err)
217+
}
218+
fi, err := f.Stat()
219+
if err != nil {
220+
utils.Fatalf("Failed to stat genesis file: %v", err)
221+
}
222+
files = append(files, fileszip.Reader{
223+
Reader: f,
224+
Size: fi.Size(),
225+
})
226+
closers = append(closers, f)
197227
}
198-
inputGenesisHash, readGenesisStore, err := genesisstore.OpenGenesisStore(genesisFile)
228+
genesisStore, genesisHashes, err := genesisstore.OpenGenesisStore(files, func() error {
229+
for _, cl := range closers {
230+
_ = cl.Close()
231+
}
232+
return nil
233+
})
199234
if err != nil {
200235
utils.Fatalf("Failed to read genesis file: %v", err)
201236
}
202237

203-
genesis = integration.InputGenesis{
204-
Hash: inputGenesisHash,
205-
Read: readGenesisStore,
206-
Close: genesisFile.Close,
238+
// check if it's a trusted preset
239+
{
240+
g := genesisStore.Genesis()
241+
gHeader := genesis.Header{
242+
GenesisID: g.GenesisID,
243+
NetworkID: g.NetworkID,
244+
NetworkName: g.NetworkName,
245+
}
246+
for _, allowed := range AllowedOperaGenesis {
247+
if allowed.Hashes.Equal(genesisHashes) && allowed.Header.Equal(gHeader) {
248+
log.Info("Genesis file is a known preset", "name", allowed.Name)
249+
goto notExperimental
250+
}
251+
}
252+
if ctx.GlobalBool(ExperimentalGenesisFlag.Name) {
253+
log.Warn("Genesis file doesn't refer to any trusted preset")
254+
} else {
255+
utils.Fatalf("Genesis file doesn't refer to any trusted preset. Enable experimental genesis with --genesis.allowExperimental")
256+
}
257+
notExperimental:
207258
}
259+
return genesisStore
208260
default:
209261
log.Crit("Network genesis is not specified")
210262
}
211-
return genesis
263+
return nil
212264
}
213265

214266
func setBootnodes(ctx *cli.Context, urls []string, cfg *node.Config) {

‎cmd/opera/launcher/consolecmd.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -77,8 +77,8 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/JavaScript-Cons
7777
func localConsole(ctx *cli.Context) error {
7878
// Create and start the node based on the CLI flags
7979
cfg := makeAllConfigs(ctx)
80-
genesis := getOperaGenesis(ctx)
81-
node, _, nodeClose := makeNode(ctx, cfg, genesis)
80+
genesisStore := getGenesisStore(ctx)
81+
node, _, nodeClose := makeNode(ctx, cfg, genesisStore)
8282
startNode(ctx, node)
8383
defer nodeClose()
8484

@@ -172,8 +172,8 @@ func dialRPC(endpoint string) (*rpc.Client, error) {
172172
func ephemeralConsole(ctx *cli.Context) error {
173173
// Create and start the node based on the CLI flags
174174
cfg := makeAllConfigs(ctx)
175-
genesis := getOperaGenesis(ctx)
176-
node, _, nodeClose := makeNode(ctx, cfg, genesis)
175+
genesisStore := getGenesisStore(ctx)
176+
node, _, nodeClose := makeNode(ctx, cfg, genesisStore)
177177
startNode(ctx, node)
178178
defer nodeClose()
179179

‎cmd/opera/launcher/consolecmd_test.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@ import (
1111
"testing"
1212
"time"
1313

14-
"github.com/Fantom-foundation/go-opera/integration/makegenesis"
15-
1614
"github.com/ethereum/go-ethereum/params"
15+
16+
"github.com/Fantom-foundation/go-opera/integration/makefakegenesis"
1717
)
1818

1919
const (
@@ -143,5 +143,5 @@ func trulyRandInt(lo, hi int) int {
143143
}
144144

145145
func genesisStart() string {
146-
return time.Unix(int64(makegenesis.FakeGenesisTime.Unix()), 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)")
146+
return time.Unix(int64(makefakegenesis.FakeGenesisTime.Unix()), 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)")
147147
}

‎cmd/opera/launcher/fake.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
"github.com/Fantom-foundation/lachesis-base/inter/idx"
1010
cli "gopkg.in/urfave/cli.v1"
1111

12-
"github.com/Fantom-foundation/go-opera/integration/makegenesis"
12+
"github.com/Fantom-foundation/go-opera/integration/makefakegenesis"
1313
)
1414

1515
// FakeNetFlag enables special testnet, where validators are automatically created
@@ -23,7 +23,7 @@ func getFakeValidatorKey(ctx *cli.Context) *ecdsa.PrivateKey {
2323
if err != nil || id == 0 {
2424
return nil
2525
}
26-
return makegenesis.FakeKey(id)
26+
return makefakegenesis.FakeKey(id)
2727
}
2828

2929
func parseFakeGen(s string) (id idx.ValidatorID, num idx.Validator, err error) {

‎cmd/opera/launcher/fake_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
"github.com/ethereum/go-ethereum/crypto"
1111
"github.com/ethereum/go-ethereum/params"
1212

13-
"github.com/Fantom-foundation/go-opera/integration/makegenesis"
13+
"github.com/Fantom-foundation/go-opera/integration/makefakegenesis"
1414
"github.com/Fantom-foundation/go-opera/inter/validatorpk"
1515
)
1616

@@ -112,7 +112,7 @@ func readFakeValidator(fakenet string) *validatorpk.PubKey {
112112
}
113113

114114
return &validatorpk.PubKey{
115-
Raw: crypto.FromECDSAPub(&makegenesis.FakeKey(n).PublicKey),
115+
Raw: crypto.FromECDSAPub(&makefakegenesis.FakeKey(n).PublicKey),
116116
Type: validatorpk.Types.Secp256k1,
117117
}
118118
}

‎cmd/opera/launcher/import.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import (
2727
"github.com/Fantom-foundation/go-opera/gossip/emitter"
2828
"github.com/Fantom-foundation/go-opera/integration"
2929
"github.com/Fantom-foundation/go-opera/inter"
30+
"github.com/Fantom-foundation/go-opera/opera/genesisstore"
3031
"github.com/Fantom-foundation/go-opera/utils/ioread"
3132
)
3233

@@ -81,7 +82,7 @@ func importEvents(ctx *cli.Context) error {
8182
}
8283

8384
// avoid P2P interaction, API calls and events emitting
84-
genesis := getOperaGenesis(ctx)
85+
genesisStore := getGenesisStore(ctx)
8586
cfg := makeAllConfigs(ctx)
8687
cfg.Opera.Protocol.EventsSemaphoreLimit.Size = math.MaxUint32
8788
cfg.Opera.Protocol.EventsSemaphoreLimit.Num = math.MaxUint32
@@ -99,16 +100,16 @@ func importEvents(ctx *cli.Context) error {
99100
cfg.Node.P2P.StaticNodes = nil
100101
cfg.Node.P2P.TrustedNodes = nil
101102

102-
err := importEventsToNode(ctx, cfg, genesis, ctx.Args()...)
103+
err := importEventsToNode(ctx, cfg, genesisStore, ctx.Args()...)
103104
if err != nil {
104105
return err
105106
}
106107

107108
return nil
108109
}
109110

110-
func importEventsToNode(ctx *cli.Context, cfg *config, genesis integration.InputGenesis, args ...string) error {
111-
node, svc, nodeClose := makeNode(ctx, cfg, genesis)
111+
func importEventsToNode(ctx *cli.Context, cfg *config, genesisStore *genesisstore.Store, args ...string) error {
112+
node, svc, nodeClose := makeNode(ctx, cfg, genesisStore)
112113
defer nodeClose()
113114
startNode(ctx, node)
114115

‎cmd/opera/launcher/launcher.go‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"github.com/Fantom-foundation/go-opera/gossip"
3030
"github.com/Fantom-foundation/go-opera/gossip/emitter"
3131
"github.com/Fantom-foundation/go-opera/integration"
32+
"github.com/Fantom-foundation/go-opera/opera/genesisstore"
3233
"github.com/Fantom-foundation/go-opera/utils/errlock"
3334
"github.com/Fantom-foundation/go-opera/valkeystore"
3435
_ "github.com/Fantom-foundation/go-opera/version"
@@ -103,6 +104,7 @@ func initFlags() {
103104
}
104105
operaFlags = []cli.Flag{
105106
GenesisFlag,
107+
ExperimentalGenesisFlag,
106108
utils.IdentityFlag,
107109
DataDirFlag,
108110
utils.MinFreeDiskSpaceFlag,
@@ -262,15 +264,15 @@ func lachesisMain(ctx *cli.Context) error {
262264
//defer tracingStop()
263265

264266
cfg := makeAllConfigs(ctx)
265-
genesisPath := getOperaGenesis(ctx)
266-
node, _, nodeClose := makeNode(ctx, cfg, genesisPath)
267+
genesisStore := getGenesisStore(ctx)
268+
node, _, nodeClose := makeNode(ctx, cfg, genesisStore)
267269
defer nodeClose()
268270
startNode(ctx, node)
269271
node.Wait()
270272
return nil
271273
}
272274

273-
func makeNode(ctx *cli.Context, cfg *config, genesis integration.InputGenesis) (*node.Node, *gossip.Service, func()) {
275+
func makeNode(ctx *cli.Context, cfg *config, genesisStore *genesisstore.Store) (*node.Node, *gossip.Service, func()) {
274276
// check errlock file
275277
errlock.SetDefaultDatadir(cfg.Node.DataDir)
276278
errlock.Check()
@@ -281,8 +283,8 @@ func makeNode(ctx *cli.Context, cfg *config, genesis integration.InputGenesis) (
281283
if err := os.MkdirAll(chaindataDir, 0700); err != nil {
282284
utils.Fatalf("Failed to create chaindata directory: %v", err)
283285
}
284-
engine, dagIndex, gdb, cdb, genesisStore, blockProc := integration.MakeEngine(integration.DBProducer(chaindataDir, cfg.cachescale), genesis, cfg.AppConfigs())
285-
_ = genesis.Close()
286+
engine, dagIndex, gdb, cdb, blockProc := integration.MakeEngine(integration.DBProducer(chaindataDir, cfg.cachescale), genesisStore.Genesis(), cfg.AppConfigs())
287+
_ = genesisStore.Close()
286288
metrics.SetDataDir(cfg.Node.DataDir)
287289

288290
valKeystore := valkeystore.NewDefaultFileKeystore(path.Join(getValKeystoreDir(cfg.Node), "validator"))

‎cmd/opera/launcher/snapshotcmd.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ func pruneState(ctx *cli.Context) error {
152152
log.Crit("DB opening error", "datadir", cfg.Node.DataDir, "err", err)
153153
}
154154

155-
if gdb.GetGenesisHash() == nil {
155+
if gdb.GetGenesisID() == nil {
156156
log.Error("Failed to open snapshot tree", "err", "genesis is not written")
157157
return err
158158
}
@@ -195,7 +195,7 @@ func verifyState(ctx *cli.Context) error {
195195
log.Crit("DB opening error", "datadir", cfg.Node.DataDir, "err", err)
196196
}
197197

198-
genesis := gdb.GetGenesisHash()
198+
genesis := gdb.GetGenesisID()
199199
if genesis == nil {
200200
log.Error("Failed to open snapshot tree", "err", "genesis is not written")
201201
return err
@@ -240,7 +240,7 @@ func traverseState(ctx *cli.Context) error {
240240
log.Crit("DB opening error", "datadir", cfg.Node.DataDir, "err", err)
241241
}
242242

243-
if gdb.GetGenesisHash() == nil {
243+
if gdb.GetGenesisID() == nil {
244244
log.Error("Failed to open snapshot tree", "err", "genesis is not written")
245245
return err
246246
}
@@ -333,7 +333,7 @@ func traverseRawState(ctx *cli.Context) error {
333333
log.Crit("DB opening error", "datadir", cfg.Node.DataDir, "err", err)
334334
}
335335

336-
if gdb.GetGenesisHash() == nil {
336+
if gdb.GetGenesisID() == nil {
337337
log.Error("Failed to open snapshot tree", "err", "genesis is not written")
338338
return err
339339
}

0 commit comments

Comments
 (0)