message:
proposals: add processing metrics for major steps
## Motivation
add information about time spent on proposal processing

diff:
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
+ "time"
"github.com/prometheus/client_golang/prometheus"
@@ -13,7 +14,6 @@ import (
"github.com/spacemeshos/go-spacemesh/log"
"github.com/spacemeshos/go-spacemesh/p2p"
"github.com/spacemeshos/go-spacemesh/p2p/pubsub"
- "github.com/spacemeshos/go-spacemesh/proposals/metrics"
"github.com/spacemeshos/go-spacemesh/sql/ballots"
"github.com/spacemeshos/go-spacemesh/sql/blocks"
"github.com/spacemeshos/go-spacemesh/sql/identities"
@@ -110,8 +110,7 @@ func NewHandler(cdb *datastore.CachedDB, f system.Fetcher, bc system.BeaconColle
// HandleProposal is the gossip receiver for Proposal.
func (h *Handler) HandleProposal(ctx context.Context, peer p2p.Peer, msg []byte) pubsub.ValidationResult {
- newCtx := log.WithNewRequestID(ctx)
- err := h.handleProposalData(newCtx, msg, peer)
+ err := h.handleProposalData(ctx, msg, peer)
switch {
case err == nil:
return pubsub.ValidationAccept
@@ -120,7 +119,7 @@ func (h *Handler) HandleProposal(ctx context.Context, peer p2p.Peer, msg []byte)
case errors.Is(err, errKnownProposal) == true:
return pubsub.ValidationIgnore
default:
- h.logger.WithContext(newCtx).With().Error("failed to process proposal gossip", log.Err(err))
+ h.logger.WithContext(ctx).With().Error("failed to process proposal gossip", log.Err(err))
return pubsub.ValidationIgnore
}
}
@@ -131,6 +130,7 @@ func (h *Handler) HandleBallotData(ctx context.Context, data []byte) error {
logger := h.logger.WithContext(newCtx)
var b types.Ballot
+ t0 := time.Now()
if err := codec.Decode(data, &b); err != nil {
logger.With().Error("malformed ballot", log.Err(err))
return errMalformedData
@@ -141,9 +141,12 @@ func (h *Handler) HandleBallotData(ctx context.Context, data []byte) error {
logger.With().Error("failed to initialize ballot", log.Err(err))
return errInitialize
}
+ ballotDuration.WithLabelValues(decodeInit).Observe(float64(time.Since(t0)))
bidHash := b.ID().AsHash32()
+ t1 := time.Now()
h.fetcher.AddPeersFromHash(bidHash, collectHashes(&b))
+ ballotDuration.WithLabelValues(peerHashes).Observe(float64(time.Since(t1)))
logger = logger.WithFields(b.ID(), b.LayerIndex)
if err := h.processBallot(ctx, &b, logger); err != nil {
@@ -174,6 +177,7 @@ func (h *Handler) HandleProposalData(ctx context.Context, data []byte) error {
func (h *Handler) handleProposalData(ctx context.Context, data []byte, peer p2p.Peer) error {
logger := h.logger.WithContext(ctx)
+ t0 := time.Now()
var p types.Proposal
if err := codec.Decode(data, &p); err != nil {
logger.With().Error("malformed proposal", log.Err(err))
@@ -185,9 +189,10 @@ func (h *Handler) handleProposalData(ctx context.Context, data []byte, peer p2p.
logger.With().Warning("failed to initialize proposal", log.Err(err))
return errInitialize
}
+ proposalDuration.WithLabelValues(decodeInit).Observe(float64(time.Since(t0)))
logger = logger.WithFields(p.ID(), p.Ballot.ID(), p.LayerIndex)
-
+ t1 := time.Now()
if has, err := proposals.Has(h.cdb, p.ID()); err != nil {
logger.With().Error("failed to look up proposal", p.ID(), log.Err(err))
return fmt.Errorf("lookup proposal %v: %w", p.ID(), err)
@@ -195,39 +200,48 @@ func (h *Handler) handleProposalData(ctx context.Context, data []byte, peer p2p.
logger.Info("known proposal")
return fmt.Errorf("%w proposal %s", errKnownProposal, p.ID())
}
+ proposalDuration.WithLabelValues(dbLookup).Observe(float64(time.Since(t1)))
- logger.With().Info("new proposal", p.ID())
-
+ logger.With().Info("new proposal", p.ID(), log.Int("num_txs", len(p.TxIDs)))
+ t2 := time.Now()
h.fetcher.RegisterPeerHashes(peer, collectHashes(&p.Ballot))
+ proposalDuration.WithLabelValues(peerHashes).Observe(float64(time.Since(t2)))
+ t3 := time.Now()
if err := h.processBallot(ctx, &p.Ballot, logger); err != nil {
logger.With().Warning("failed to process ballot", log.Err(err))
return err
}
+ proposalDuration.WithLabelValues(ballot).Observe(float64(time.Since(t3)))
+ t4 := time.Now()
if err := h.checkTransactions(ctx, &p); err != nil {
logger.With().Warning("failed to fetch proposal TXs", log.Err(err))
return err
}
+ proposalDuration.WithLabelValues(fetchTXs).Observe(float64(time.Since(t4)))
logger.With().Debug("proposal is syntactically valid")
-
+ t5 := time.Now()
if err := proposals.Add(h.cdb, &p); err != nil {
logger.With().Error("failed to save proposal", log.Err(err))
return fmt.Errorf("save proposal: %w", err)
}
-
+ proposalDuration.WithLabelValues(dbSave).Observe(float64(time.Since(t5)))
logger.With().Info("added proposal to database", p.ID())
+ t6 := time.Now()
if err := h.mesh.AddTXsFromProposal(ctx, p.LayerIndex, p.ID(), p.TxIDs); err != nil {
return fmt.Errorf("proposal add TXs: %w", err)
}
+ proposalDuration.WithLabelValues(linkTxs).Observe(float64(time.Since(t6)))
reportProposalMetrics(&p)
return nil
}
func (h *Handler) processBallot(ctx context.Context, b *types.Ballot, logger log.Log) error {
+ t0 := time.Now()
if has, err := ballots.Has(h.cdb, b.ID()); err != nil {
h.logger.WithContext(ctx).With().Error("failed to look up ballot", b.ID(), log.Err(err))
return fmt.Errorf("lookup ballot %v: %w", b.ID(), err)
@@ -235,6 +249,7 @@ func (h *Handler) processBallot(ctx context.Context, b *types.Ballot, logger log
logger.Debug("known ballot", b.ID())
return nil
}
+ ballotDuration.WithLabelValues(dbLookup).Observe(float64(time.Since(t0)))
logger.With().Info("new ballot", log.Inline(b))
@@ -243,36 +258,46 @@ func (h *Handler) processBallot(ctx context.Context, b *types.Ballot, logger log
return fmt.Errorf("syntactic-check ballot: %w", err)
}
+ t1 := time.Now()
if err := ballots.Add(h.cdb, b); err != nil {
return fmt.Errorf("save ballot: %w", err)
}
+ ballotDuration.WithLabelValues(dbSave).Observe(float64(time.Since(t1)))
- reportBallotMetrics(b)
+ reportVotesMetrics(b)
return nil
}
func (h *Handler) checkBallotSyntacticValidity(ctx context.Context, b *types.Ballot) error {
h.logger.WithContext(ctx).With().Debug("checking proposal syntactic validity")
+ t0 := time.Now()
if err := h.checkBallotDataIntegrity(b); err != nil {
h.logger.WithContext(ctx).With().Warning("ballot integrity check failed", log.Err(err))
return err
}
+ ballotDuration.WithLabelValues(dataCheck).Observe(float64(time.Since(t0)))
+ t1 := time.Now()
if err := h.checkBallotDataAvailability(ctx, b); err != nil {
h.logger.WithContext(ctx).With().Warning("ballot data availability check failed", log.Err(err))
return err
}
+ ballotDuration.WithLabelValues(fetchRef).Observe(float64(time.Since(t1)))
+ t2 := time.Now()
if err := h.checkVotesConsistency(ctx, b); err != nil {
h.logger.WithContext(ctx).With().Warning("ballot votes consistency check failed", log.Err(err))
return err
}
+ ballotDuration.WithLabelValues(votes).Observe(float64(time.Since(t2)))
+ t3 := time.Now()
if eligible, err := h.validator.CheckEligibility(ctx, b); err != nil || !eligible {
h.logger.WithContext(ctx).With().Warning("ballot eligibility check failed", log.Err(err))
return errNotEligible
}
+ ballotDuration.WithLabelValues(eligible).Observe(float64(time.Since(t3)))
h.logger.WithContext(ctx).With().Debug("ballot is syntactically valid")
return nil
@@ -429,7 +454,6 @@ func (h *Handler) checkBallotDataAvailability(ctx context.Context, b *types.Ball
return fmt.Errorf("fetch blocks: %w", err)
}
}
-
return nil
}
@@ -463,12 +487,12 @@ func (h *Handler) checkTransactions(ctx context.Context, p *types.Proposal) erro
}
func reportProposalMetrics(p *types.Proposal) {
- metrics.ProposalSize.WithLabelValues().Observe(float64(len(p.Bytes())))
- metrics.NumTxsInProposal.WithLabelValues().Observe(float64(len(p.TxIDs)))
+ proposalSize.WithLabelValues().Observe(float64(len(p.Bytes())))
+ numTxsInProposal.WithLabelValues().Observe(float64(len(p.TxIDs)))
}
-func reportBallotMetrics(b *types.Ballot) {
- metrics.NumBlocksInException.With(prometheus.Labels{metrics.DiffTypeLabel: metrics.DiffTypeAgainst}).Observe(float64(len(b.Votes.Against)))
- metrics.NumBlocksInException.With(prometheus.Labels{metrics.DiffTypeLabel: metrics.DiffTypeFor}).Observe(float64(len(b.Votes.Support)))
- metrics.NumBlocksInException.With(prometheus.Labels{metrics.DiffTypeLabel: metrics.DiffTypeNeutral}).Observe(float64(len(b.Votes.Abstain)))
+func reportVotesMetrics(b *types.Ballot) {
+ numBlocksInException.With(prometheus.Labels{diffTypeLabel: diffTypeAgainst}).Observe(float64(len(b.Votes.Against)))
+ numBlocksInException.With(prometheus.Labels{diffTypeLabel: diffTypeFor}).Observe(float64(len(b.Votes.Support)))
+ numBlocksInException.With(prometheus.Labels{diffTypeLabel: diffTypeNeutral}).Observe(float64(len(b.Votes.Abstain)))
}

diff:
+package proposals
+
+import (
+ "github.com/prometheus/client_golang/prometheus"
+
+ "github.com/spacemeshos/go-spacemesh/metrics"
+)
+
+const (
+ subsystem = "proposals"
+
+ // diffTypeLabel is the label name for the type of different opinions with base block.
+ diffTypeLabel = "diff_type"
+
+ // diffTypeFor is the label value for different opinions with base ballot when voting for blocks.
+ diffTypeFor = "diff_for"
+ // diffTypeAgainst is the label value for different opinions with base ballot when voting against blocks.
+ diffTypeAgainst = "diff_against"
+ // diffTypeNeutral is the label value for different opinions with base ballot when voting neutral on blocks.
+ diffTypeNeutral = "diff_neutral"
+)
+
+// proposalSize records average size of proposals.
+var proposalSize = metrics.NewHistogramWithBuckets(
+ "proposal_size",
+ subsystem,
+ "proposal size in bytes",
+ []string{},
+ prometheus.ExponentialBuckets(100, 2, 8),
+)
+
+// numTxsInProposal records average number of transactions in a proposal.
+var numTxsInProposal = metrics.NewHistogramWithBuckets(
+ "num_txs_in_proposal",
+ subsystem,
+ "number of transactions in proposal",
+ []string{},
+ prometheus.ExponentialBuckets(1, 2, 8),
+)
+
+// numBlocksInException records the number of blocks encoded in a ballot for a given exception type (against, for, and neutral).
+var numBlocksInException = metrics.NewHistogramWithBuckets(
+ "num_blocks_in_exception",
+ subsystem,
+ "number of blocks in an exception list",
+ []string{
+ diffTypeLabel,
+ },
+ prometheus.ExponentialBuckets(1, 2, 8),
+)
+
+const (
+ // labels for each step of proposal/ballot processing.
+ decodeInit = "decode"
+ dbLookup = "db_r"
+ peerHashes = "hashes"
+ ballot = "ballot"
+ fetchTXs = "txs"
+ dbSave = "db_w"
+ linkTxs = "link"
+
+ dataCheck = "data" // check data integrity
+ fetchRef = "fetch" // check referenced ballots/atxs/blocks are available
+ votes = "votes" // check votes are consistent
+ eligible = "elig" // check ballot eligibility
+)
+
+var (
+ proposalDuration = metrics.NewHistogramWithBuckets(
+ "proposal_duration",
+ subsystem,
+ "Duration in ns for processing a proposal",
+ []string{"step"},
+ prometheus.ExponentialBuckets(1_000_000, 4, 10),
+ )
+ ballotDuration = metrics.NewHistogramWithBuckets(
+ "ballot_duration",
+ subsystem,
+ "Duration in ns for processing a ballot",
+ []string{"step"},
+ prometheus.ExponentialBuckets(1_000_000, 4, 10),
+ )
+)

diff:
-package metrics
-
-import (
- "github.com/prometheus/client_golang/prometheus"
-
- "github.com/spacemeshos/go-spacemesh/metrics"
-)
-
-const (
- subsystem = "proposals"
-
- // DiffTypeLabel is the label name for the type of different opinions with base block.
- DiffTypeLabel = "diff_type"
-
- // DiffTypeFor is the label value for different opinions with base ballot when voting for blocks.
- DiffTypeFor = "diff_for"
- // DiffTypeAgainst is the label value for different opinions with base ballot when voting against blocks.
- DiffTypeAgainst = "diff_against"
- // DiffTypeNeutral is the label value for different opinions with base ballot when voting neutral on blocks.
- DiffTypeNeutral = "diff_neutral"
-)
-
-// ProposalSize records average size of proposals.
-var ProposalSize = metrics.NewHistogramWithBuckets(
- "proposal_size",
- subsystem,
- "proposal size in bytes",
- []string{},
- prometheus.ExponentialBuckets(100, 2, 8),
-)
-
-// NumTxsInProposal records average number of transactions in a proposal.
-var NumTxsInProposal = metrics.NewHistogramWithBuckets(
- "num_txs_in_proposal",
- subsystem,
- "number of transactions in proposal",
- []string{},
- prometheus.ExponentialBuckets(1, 2, 8),
-)
-
-// NumBlocksInException records the number of blocks encoded in a ballot for a given exception type (against, for, and neutral).
-var NumBlocksInException = metrics.NewHistogramWithBuckets(
- "num_blocks_in_exception",
- subsystem,
- "number of blocks in an exception list",
- []string{
- DiffTypeLabel,
- },
- prometheus.ExponentialBuckets(1, 2, 8),
-)

old_path: systest/Makefile

new_path: systest/Makefile

diff:
@@ -32,7 +32,7 @@ run:
--labels="testid=$(test_id)" \
--image-pull-policy=IfNotPresent -- \
tests -test.v -test.timeout=0 -test.run=$(test_name) -namespace=$(namespace) -clusters=$(clusters) -storage=$(storage) -size=$(size) -image=$(smesher_image) -level=$(level) -node-selector=$(node_selector) -bootstrap=$(bootstrap) -keep=$(keep) -testid=$(test_id) -labels=$(label)
- -@kubectl wait --timeout=5s --for=condition=ready pod/$(test_pod_name)
+ -@kubectl wait --timeout=10s --for=condition=ready pod/$(test_pod_name)
@kubectl logs $(test_pod_name) -f
.PHONY: clean

license: MIT License

repo: spacemeshos/go-spacemesh

original_message:
proposals: add processing metrics for major steps (#3369)

## Motivation
add information about time spent on proposal processing


message:
documentation: add comments about weight and RatNum
## Motivation
<!-- Please mention the issue fixed by this PR or detailed motivation -->
Closes
<!-- `Closes #XXXX, closes #XXXX, ...` links mentioned issues to this PR and automatically closes them when this it's merged -->

change_type: MODIFY

diff:
@@ -55,11 +55,14 @@ type InnerBlock struct {
}
// RatNum represents a rational number with the numerator and denominator.
+// note: RatNum aims to be a generic representation of a rational number and parse-able by
+// different programming languages.
+// for doing math around weight inside go-spacemesh codebase, use util.Weight.
type RatNum struct {
Num, Denom uint64
}
-// AnyReward contains the rewards inforamtion.
+// AnyReward contains the reward information.
type AnyReward struct {
Coinbase Address
Weight RatNum

diff:
@@ -25,6 +25,8 @@ func WeightFromNumDenom(num, denom uint64) Weight {
}
// Weight represents weight for any ATX/ballot.
+// note: this is golang specific and is used to do math on weight.
+// for representing weight over the wire or data persistence, use types.RatNum.
type Weight struct {
*big.Rat
}

original_message:
documentation: add comments about weight and RatNum (#3433)

## Motivation
<!-- Please mention the issue fixed by this PR or detailed motivation -->
Closes #3393
<!-- `Closes #XXXX, closes #XXXX, ...` links mentioned issues to this PR and automatically closes them when this it's merged -->


message:
fix flaky tests against time
## Motivation
fix flaky tests
tested with
```
go clean -testcache && go test ./activation/ -v -run "^TestBuilder_waitForFirstATX" -count 2000
go clean -testcache && go test ./timesync/ -v -run "^TestClock_" -count 1000
```

diff:
@@ -321,16 +321,16 @@ func (b *Builder) generateProof() error {
return nil
}
-func (b *Builder) waitForFirstATX(ctx context.Context) {
+func (b *Builder) waitForFirstATX(ctx context.Context) bool {
currentLayer := b.layerClock.GetCurrentLayer()
currEpoch := currentLayer.GetEpoch()
if currEpoch == 0 { // genesis miner
- return
+ return false
}
if prev, err := b.cdb.GetPrevAtx(b.nodeID); err == nil {
if prev.PubLayerID.GetEpoch() == currEpoch {
// miner has ATX in previous epoch
- return
+ return false
}
}
@@ -349,7 +349,7 @@ func (b *Builder) waitForFirstATX(ctx context.Context) {
log.Stringer("wait_till_epoch", waitTill.GetEpoch()))
select {
case <-ctx.Done():
- return
+ return false
case <-b.layerClock.AwaitLayer(waitTill):
wait = window
}
@@ -360,12 +360,13 @@ func (b *Builder) waitForFirstATX(ctx context.Context) {
defer timer.Stop()
select {
case <-ctx.Done():
- return
+ return false
case <-timer.C:
}
b.log.WithContext(ctx).With().Info("ready to build first atx",
log.Stringer("current_layer", b.layerClock.GetCurrentLayer()),
log.Stringer("current_epoch", b.layerClock.GetCurrentLayer().GetEpoch()))
+ return true
}
// loop is the main loop that tries to create an atx per tick received from the global clock.

diff:
@@ -329,23 +329,13 @@ func TestBuilder_waitForFirstATX(t *testing.T) {
WithPoetConfig(poetCfg))
b.initialPost = initialPost
- layerStart := time.Now().Add(-1 * time.Millisecond)
- poetWindwow := poetCfg.PhaseShift - poetCfg.CycleGap + poetCfg.GracePeriod
- expectedWait := poetWindwow - time.Millisecond
ch := make(chan struct{}, 1)
close(ch)
- current := types.NewLayerID(layersPerEpoch * 2)
+ current := types.NewLayerID(layersPerEpoch * 2) // first layer of epoch 2
addPrevAtx(t, cdb, current.GetEpoch()-1)
mClock.EXPECT().GetCurrentLayer().Return(current).AnyTimes()
- start := time.Time{}
- mClock.EXPECT().LayerToTime(current).DoAndReturn(
- func(_ types.LayerID) time.Time {
- start = time.Now()
- return layerStart
- })
- b.waitForFirstATX(context.TODO())
- elapsed := time.Since(start)
- require.GreaterOrEqual(t, elapsed, expectedWait)
+ mClock.EXPECT().LayerToTime(current).Return(time.Now().Add(100 * time.Millisecond))
+ require.True(t, b.waitForFirstATX(context.TODO()))
}
func TestBuilder_waitForFirstATX_nextEpoch(t *testing.T) {
@@ -369,23 +359,15 @@ func TestBuilder_waitForFirstATX_nextEpoch(t *testing.T) {
WithPoetConfig(poetCfg))
b.initialPost = initialPost
- poetWindwow := poetCfg.PhaseShift - poetCfg.CycleGap + poetCfg.GracePeriod
ch := make(chan struct{}, 1)
close(ch)
- current := types.NewLayerID(layersPerEpoch * 2)
+ current := types.NewLayerID(layersPerEpoch * 2) // first layer of epoch 2
addPrevAtx(t, cdb, current.GetEpoch()-1)
mClock.EXPECT().GetCurrentLayer().Return(current)
mClock.EXPECT().LayerToTime(current).Return(time.Now().Add(-5 * time.Millisecond))
- start := time.Time{}
- mClock.EXPECT().AwaitLayer(current.Add(layersPerEpoch)).DoAndReturn(
- func(_ types.LayerID) chan struct{} {
- start = time.Now()
- return ch
- })
+ mClock.EXPECT().AwaitLayer(current.Add(layersPerEpoch)).Return(ch)
mClock.EXPECT().GetCurrentLayer().Return(current.Add(layersPerEpoch)).AnyTimes()
- b.waitForFirstATX(context.TODO())
- elapsed := time.Since(start)
- require.GreaterOrEqual(t, elapsed, poetWindwow)
+ require.True(t, b.waitForFirstATX(context.TODO()))
}
func TestBuilder_waitForFirstATX_Genesis(t *testing.T) {
@@ -397,7 +379,7 @@ func TestBuilder_waitForFirstATX_Genesis(t *testing.T) {
current := types.NewLayerID(0)
mClock.EXPECT().GetCurrentLayer().Return(current)
- b.waitForFirstATX(context.TODO())
+ require.False(t, b.waitForFirstATX(context.TODO()))
}
func TestBuilder_waitForFirstATX_NoWait(t *testing.T) {
@@ -410,7 +392,7 @@ func TestBuilder_waitForFirstATX_NoWait(t *testing.T) {
current := types.NewLayerID(layersPerEpoch)
addPrevAtx(t, cdb, current.GetEpoch())
mClock.EXPECT().GetCurrentLayer().Return(current)
- b.waitForFirstATX(context.TODO())
+ require.False(t, b.waitForFirstATX(context.TODO()))
}
func TestBuilder_StartSmeshingCoinbase(t *testing.T) {

diff:
@@ -58,8 +58,6 @@ func NewClock(c Clock, tickInterval time.Duration, genesisTime time.Time, logger
func (t *TimeClock) startClock() error {
t.log.Info("starting global clock now=%v genesis=%v %p", t.clock.Now(), t.genesis, t)
- tmr := time.NewTimer(0)
- defer tmr.Stop()
for {
currLayer := t.Ticker.TimeToLayer(t.clock.Now()) // get current layer
nextLayer := currLayer.Add(1)
@@ -68,13 +66,14 @@ func (t *TimeClock) startClock() error {
}
nextTickTime := t.Ticker.LayerToTime(nextLayer) // get next tick time for the next layer
diff := nextTickTime.Sub(t.clock.Now())
- tmr.Reset(diff)
t.log.With().Info("global clock going to sleep before next layer",
log.String("diff", diff.String()),
log.FieldNamed("curr_layer", currLayer),
log.FieldNamed("next_layer", nextLayer))
+ tmr := time.NewTimer(diff)
select {
case <-tmr.C:
+ tmr.Stop()
t.mu.Lock()
subscriberCount := len(t.subscribers)
t.mu.Unlock()
@@ -90,6 +89,7 @@ func (t *TimeClock) startClock() error {
log.Int("missed", missed))
}
case <-t.stop:
+ tmr.Stop()
t.log.Info("stopping global clock %p", t)
return nil
}

original_message:
fix flaky tests against time (#3521)

## Motivation
fix flaky tests

tested with
```
go clean -testcache && go test ./activation/ -v -run "^TestBuilder_waitForFirstATX" -count 2000
go clean -testcache && go test ./timesync/ -v -run "^TestClock_" -count 1000
```


message:
add back missing call to mesh.AddBallot() when receiving proposal/ballot
## Motivation
the call was removed by mistake from :-(

diff:
@@ -275,7 +275,7 @@ func (h *Handler) processBallot(ctx context.Context, logger log.Log, b *types.Ba
}
t1 := time.Now()
- if err := ballots.Add(h.cdb, b); err != nil {
+ if err := h.mesh.AddBallot(b); err != nil {
if errors.Is(err, sql.ErrObjectExists) {
return fmt.Errorf("%w: ballot %s", errKnownBallot, b.ID())
}

diff:
@@ -139,15 +139,8 @@ func checkProposal(t *testing.T, cdb *datastore.CachedDB, p *types.Proposal, exi
}
}
-func checkBallot(t *testing.T, cdb *datastore.CachedDB, b *types.Ballot, exist, malicious bool) {
+func checkIdentity(t *testing.T, cdb *datastore.CachedDB, b *types.Ballot, malicious bool) {
t.Helper()
- gotB, err := ballots.Get(cdb, b.ID())
- if exist {
- require.NoError(t, err)
- require.Equal(t, b, gotB)
- } else {
- require.ErrorIs(t, err, sql.ErrNotFound)
- }
gotM, err := identities.IsMalicious(cdb, b.SmesherID().Bytes())
require.NoError(t, err)
require.Equal(t, malicious, gotM)
@@ -256,7 +249,6 @@ func TestBallot_NotRefBallotButHasEpochData(t *testing.T) {
data := encodeBallot(t, b)
th.mf.EXPECT().AddPeersFromHash(b.ID().AsHash32(), collectHashes(*b))
require.ErrorIs(t, th.HandleSyncedBallot(context.TODO(), data), errUnexpectedEpochData)
- checkBallot(t, th.cdb, b, false, false)
}
func TestBallot_BallotDoubleVotedWithinHdist(t *testing.T) {
@@ -274,7 +266,7 @@ func TestBallot_BallotDoubleVotedWithinHdist(t *testing.T) {
th.mf.EXPECT().GetAtxs(gomock.Any(), types.ATXIDList{b.AtxID}).Return(nil).Times(1)
th.mf.EXPECT().GetBlocks(gomock.Any(), b.Votes.Support).Return(nil).Times(1)
require.ErrorIs(t, th.HandleSyncedBallot(context.TODO(), data), errDoubleVoting)
- checkBallot(t, th.cdb, b, false, true)
+ checkIdentity(t, th.cdb, b, true)
}
func TestBallot_BallotDoubleVotedWithinHdist_LyrBfrHdist(t *testing.T) {
@@ -293,7 +285,7 @@ func TestBallot_BallotDoubleVotedWithinHdist_LyrBfrHdist(t *testing.T) {
th.mf.EXPECT().GetAtxs(gomock.Any(), types.ATXIDList{b.AtxID}).Return(nil).Times(1)
th.mf.EXPECT().GetBlocks(gomock.Any(), b.Votes.Support).Return(nil).Times(1)
require.ErrorIs(t, th.HandleSyncedBallot(context.TODO(), data), errDoubleVoting)
- checkBallot(t, th.cdb, b, false, true)
+ checkIdentity(t, th.cdb, b, true)
}
func TestBallot_BallotDoubleVotedOutsideHdist(t *testing.T) {
@@ -315,10 +307,9 @@ func TestBallot_BallotDoubleVotedOutsideHdist(t *testing.T) {
require.Equal(t, b.ID(), ballot.ID())
return true, nil
})
+ th.mm.EXPECT().AddBallot(b).Return(nil)
require.NoError(t, th.HandleSyncedBallot(context.TODO(), data))
- got, err := ballots.Get(th.cdb, b.ID())
- require.NoError(t, err)
- require.Equal(t, b, got)
+ checkIdentity(t, th.cdb, b, false)
}
func TestBallot_ConflictingForAndAgainst(t *testing.T) {
@@ -336,7 +327,7 @@ func TestBallot_ConflictingForAndAgainst(t *testing.T) {
th.mf.EXPECT().GetAtxs(gomock.Any(), types.ATXIDList{b.AtxID}).Return(nil).Times(1)
th.mf.EXPECT().GetBlocks(gomock.Any(), append(b.Votes.Support, b.Votes.Against...)).Return(nil).Times(1)
require.ErrorIs(t, th.HandleSyncedBallot(context.TODO(), data), errConflictingExceptions)
- checkBallot(t, th.cdb, b, false, true)
+ checkIdentity(t, th.cdb, b, true)
}
func TestBallot_ConflictingForAndAbstain(t *testing.T) {
@@ -354,7 +345,7 @@ func TestBallot_ConflictingForAndAbstain(t *testing.T) {
th.mf.EXPECT().GetAtxs(gomock.Any(), types.ATXIDList{b.AtxID}).Return(nil).Times(1)
th.mf.EXPECT().GetBlocks(gomock.Any(), b.Votes.Support).Return(nil).Times(1)
require.ErrorIs(t, th.HandleSyncedBallot(context.TODO(), data), errConflictingExceptions)
- checkBallot(t, th.cdb, b, false, true)
+ checkIdentity(t, th.cdb, b, true)
}
func TestBallot_ConflictingAgainstAndAbstain(t *testing.T) {
@@ -374,7 +365,7 @@ func TestBallot_ConflictingAgainstAndAbstain(t *testing.T) {
th.mf.EXPECT().GetAtxs(gomock.Any(), types.ATXIDList{b.AtxID}).Return(nil).Times(1)
th.mf.EXPECT().GetBlocks(gomock.Any(), append(b.Votes.Support, b.Votes.Against...)).Return(nil).Times(1)
require.ErrorIs(t, th.HandleSyncedBallot(context.TODO(), data), errConflictingExceptions)
- checkBallot(t, th.cdb, b, false, true)
+ checkIdentity(t, th.cdb, b, true)
}
func TestBallot_ExceedMaxExceptions(t *testing.T) {
@@ -392,7 +383,7 @@ func TestBallot_ExceedMaxExceptions(t *testing.T) {
th.mf.EXPECT().GetAtxs(gomock.Any(), types.ATXIDList{b.AtxID}).Return(nil).Times(1)
th.mf.EXPECT().GetBlocks(gomock.Any(), b.Votes.Support).Return(nil).Times(1)
require.ErrorIs(t, th.HandleSyncedBallot(context.TODO(), data), errExceptionsOverflow)
- checkBallot(t, th.cdb, b, false, false)
+ checkIdentity(t, th.cdb, b, false)
}
func TestBallot_BallotsNotAvailable(t *testing.T) {
@@ -404,7 +395,7 @@ func TestBallot_BallotsNotAvailable(t *testing.T) {
th.mf.EXPECT().AddPeersFromHash(b.ID().AsHash32(), collectHashes(*b))
th.mf.EXPECT().GetBallots(gomock.Any(), []types.BallotID{b.Votes.Base, b.RefBallot}).Return(errUnknown).Times(1)
require.ErrorIs(t, th.HandleSyncedBallot(context.TODO(), data), errUnknown)
- checkBallot(t, th.cdb, b, false, false)
+ checkIdentity(t, th.cdb, b, false)
}
func TestBallot_ATXsNotAvailable(t *testing.T) {
@@ -416,7 +407,7 @@ func TestBallot_ATXsNotAvailable(t *testing.T) {
errUnknown := errors.New("unknown")
th.mf.EXPECT().GetAtxs(gomock.Any(), types.ATXIDList{b.AtxID}).Return(errUnknown).Times(1)
require.ErrorIs(t, th.HandleSyncedBallot(context.TODO(), data), errUnknown)
- checkBallot(t, th.cdb, b, false, false)
+ checkIdentity(t, th.cdb, b, false)
}
func TestBallot_BlocksNotAvailable(t *testing.T) {
@@ -429,7 +420,7 @@ func TestBallot_BlocksNotAvailable(t *testing.T) {
errUnknown := errors.New("unknown")
th.mf.EXPECT().GetBlocks(gomock.Any(), b.Votes.Support).Return(errUnknown).Times(1)
require.ErrorIs(t, th.HandleSyncedBallot(context.TODO(), data), errUnknown)
- checkBallot(t, th.cdb, b, false, false)
+ checkIdentity(t, th.cdb, b, false)
}
func TestBallot_ErrorCheckingEligible(t *testing.T) {
@@ -451,7 +442,7 @@ func TestBallot_ErrorCheckingEligible(t *testing.T) {
return false, errors.New("unknown")
})
require.ErrorIs(t, th.HandleSyncedBallot(context.TODO(), data), errNotEligible)
- checkBallot(t, th.cdb, b, false, false)
+ checkIdentity(t, th.cdb, b, false)
}
func TestBallot_NotEligible(t *testing.T) {
@@ -473,7 +464,7 @@ func TestBallot_NotEligible(t *testing.T) {
return false, nil
})
require.ErrorIs(t, th.HandleSyncedBallot(context.TODO(), data), errNotEligible)
- checkBallot(t, th.cdb, b, false, false)
+ checkIdentity(t, th.cdb, b, false)
}
func TestBallot_Success(t *testing.T) {
@@ -493,8 +484,9 @@ func TestBallot_Success(t *testing.T) {
require.Equal(t, b.ID(), ballot.ID())
return true, nil
})
+ th.mm.EXPECT().AddBallot(b).Return(nil)
require.NoError(t, th.HandleSyncedBallot(context.TODO(), data))
- checkBallot(t, th.cdb, b, true, false)
+ checkIdentity(t, th.cdb, b, false)
}
func TestBallot_RefBallot(t *testing.T) {
@@ -517,8 +509,9 @@ func TestBallot_RefBallot(t *testing.T) {
require.Equal(t, b.ID(), ballot.ID())
return true, nil
})
+ th.mm.EXPECT().AddBallot(b).Return(nil)
require.NoError(t, th.HandleSyncedBallot(context.TODO(), data))
- checkBallot(t, th.cdb, b, true, false)
+ checkIdentity(t, th.cdb, b, false)
}
func TestProposal_MalformedData(t *testing.T) {
@@ -579,6 +572,11 @@ func TestProposal_DuplicateTXs(t *testing.T) {
require.Equal(t, p.Ballot.ID(), ballot.ID())
return true, nil
})
+ th.mm.EXPECT().AddBallot(&p.Ballot).Return(nil).DoAndReturn(
+ func(got *types.Ballot) error {
+ require.NoError(t, ballots.Add(th.cdb, got))
+ return nil
+ })
th.mf.EXPECT().RegisterPeerHashes(p2p.NoPeer, collectHashes(*p))
require.ErrorIs(t, th.HandleSyncedProposal(context.TODO(), data), errDuplicateTX)
checkProposal(t, th.cdb, p, false)
@@ -601,6 +599,11 @@ func TestProposal_TXsNotAvailable(t *testing.T) {
require.Equal(t, p.Ballot.ID(), ballot.ID())
return true, nil
})
+ th.mm.EXPECT().AddBallot(&p.Ballot).Return(nil).DoAndReturn(
+ func(got *types.Ballot) error {
+ require.NoError(t, ballots.Add(th.cdb, got))
+ return nil
+ })
errUnknown := errors.New("unknown")
th.mf.EXPECT().GetProposalTxs(gomock.Any(), p.TxIDs).Return(errUnknown).Times(1)
@@ -624,6 +627,11 @@ func TestProposal_FailedToAddProposalTXs(t *testing.T) {
require.Equal(t, p.Ballot.ID(), ballot.ID())
return true, nil
})
+ th.mm.EXPECT().AddBallot(&p.Ballot).Return(nil).DoAndReturn(
+ func(got *types.Ballot) error {
+ require.NoError(t, ballots.Add(th.cdb, got))
+ return nil
+ })
th.mf.EXPECT().GetProposalTxs(gomock.Any(), p.TxIDs).Return(nil).Times(1)
th.mf.EXPECT().RegisterPeerHashes(p2p.NoPeer, collectHashes(*p))
errUnknown := errors.New("unknown")
@@ -650,6 +658,11 @@ func TestProposal_ProposalGossip_Concurrent(t *testing.T) {
require.Equal(t, p.Ballot.ID(), ballot.ID())
return true, nil
}).MinTimes(1).MaxTimes(2)
+ th.mm.EXPECT().AddBallot(&p.Ballot).Return(nil).DoAndReturn(
+ func(got *types.Ballot) error {
+ _ = ballots.Add(th.cdb, got)
+ return nil
+ }).MinTimes(1).MaxTimes(2)
th.mf.EXPECT().GetProposalTxs(gomock.Any(), p.TxIDs).Return(nil).MinTimes(1).MaxTimes(2)
th.mm.EXPECT().AddTXsFromProposal(gomock.Any(), p.LayerIndex, p.ID(), p.TxIDs).Return(nil).Times(1)
@@ -708,12 +721,14 @@ func TestProposal_ProposalGossip_Fetched(t *testing.T) {
if tc.propFetched {
// a separate goroutine fetched the propFetched and saved it to database
require.NoError(t, proposals.Add(th.cdb, p))
+ require.NoError(t, ballots.Add(th.cdb, &p.Ballot))
} else {
// a separate goroutine fetched the ballot and saved it to database
require.NoError(t, ballots.Add(th.cdb, &p.Ballot))
}
return true, nil
})
+ th.mm.EXPECT().AddBallot(&p.Ballot).Return(nil).Return(nil)
th.mf.EXPECT().GetProposalTxs(gomock.Any(), p.TxIDs).Return(nil)
if tc.propFetched {
require.Equal(t, pubsub.ValidationIgnore, th.HandleProposal(context.TODO(), p2p.NoPeer, data))
@@ -722,7 +737,7 @@ func TestProposal_ProposalGossip_Fetched(t *testing.T) {
require.Equal(t, pubsub.ValidationAccept, th.HandleProposal(context.TODO(), p2p.NoPeer, data))
}
checkProposal(t, th.cdb, p, true)
- checkBallot(t, th.cdb, &p.Ballot, true, false)
+ checkIdentity(t, th.cdb, &p.Ballot, false)
})
}
}
@@ -743,6 +758,11 @@ func TestProposal_ValidProposal(t *testing.T) {
require.Equal(t, p.Ballot.ID(), ballot.ID())
return true, nil
})
+ th.mm.EXPECT().AddBallot(&p.Ballot).Return(nil).DoAndReturn(
+ func(got *types.Ballot) error {
+ require.NoError(t, ballots.Add(th.cdb, got))
+ return nil
+ })
th.mf.EXPECT().GetProposalTxs(gomock.Any(), p.TxIDs).Return(nil).Times(1)
th.mf.EXPECT().RegisterPeerHashes(p2p.NoPeer, collectHashes(*p))
th.mm.EXPECT().AddTXsFromProposal(gomock.Any(), p.LayerIndex, p.ID(), p.TxIDs).Return(nil).Times(1)
@@ -766,6 +786,11 @@ func TestMetrics(t *testing.T) {
require.Equal(t, p.Ballot.ID(), ballot.ID())
return true, nil
})
+ th.mm.EXPECT().AddBallot(&p.Ballot).Return(nil).DoAndReturn(
+ func(got *types.Ballot) error {
+ require.NoError(t, ballots.Add(th.cdb, got))
+ return nil
+ })
th.mf.EXPECT().GetProposalTxs(gomock.Any(), p.TxIDs).Return(nil).Times(1)
th.mf.EXPECT().RegisterPeerHashes(p2p.NoPeer, collectHashes(*p))
th.mm.EXPECT().AddTXsFromProposal(gomock.Any(), p.LayerIndex, p.ID(), p.TxIDs).Return(nil).Times(1)

diff:
@@ -9,6 +9,7 @@ import (
//go:generate mockgen -package=mocks -destination=./mocks/mocks.go -source=./interface.go
type meshProvider interface {
+ AddBallot(*types.Ballot) error
AddTXsFromProposal(context.Context, types.LayerID, types.ProposalID, []types.TransactionID) error
}

diff:
@@ -35,6 +35,20 @@ func (m *MockmeshProvider) EXPECT() *MockmeshProviderMockRecorder {
return m.recorder
}
+// AddBallot mocks base method.
+func (m *MockmeshProvider) AddBallot(arg0 *types.Ballot) error {
+ m.ctrl.T.Helper()
+ ret := m.ctrl.Call(m, "AddBallot", arg0)
+ ret0, _ := ret[0].(error)
+ return ret0
+}
+
+// AddBallot indicates an expected call of AddBallot.
+func (mr *MockmeshProviderMockRecorder) AddBallot(arg0 interface{}) *gomock.Call {
+ mr.mock.ctrl.T.Helper()
+ return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddBallot", reflect.TypeOf((*MockmeshProvider)(nil).AddBallot), arg0)
+}
+
// AddTXsFromProposal mocks base method.
func (m *MockmeshProvider) AddTXsFromProposal(arg0 context.Context, arg1 types.LayerID, arg2 types.ProposalID, arg3 []types.TransactionID) error {
m.ctrl.T.Helper()

original_message:
add back missing call to mesh.AddBallot() when receiving proposal/ballot (#3542)

## Motivation
the call was removed by mistake from https://github.com/spacemeshos/go-spacemesh/pull/3265 :-(


message:
block certifier: fix certifier concurrent map access
## Motivation
race condition, added test that failed with -race -count 5 and pass after the fix

diff:
@@ -298,6 +298,8 @@ func (c *Certifier) HandleSyncedCertificate(ctx context.Context, lid types.Layer
}
func (c *Certifier) certified(lid types.LayerID, bid types.BlockID) bool {
+ c.mu.Lock()
+ defer c.mu.Unlock()
if _, ok := c.certifyMsgs[lid]; ok {
if _, ok := c.certifyMsgs[lid][bid]; ok {
return c.certifyMsgs[lid][bid].done

diff:
@@ -3,6 +3,7 @@ package blocks
import (
"context"
"errors"
+ "sync"
"testing"
"github.com/golang/mock/gomock"
@@ -231,24 +232,56 @@ func Test_HandleCertifyMessage(t *testing.T) {
}
func Test_HandleCertifyMessage_Certified(t *testing.T) {
- tc := newTestCertifier(t)
- numMsgs := tc.cfg.CommitteeSize
- cutoff := tc.cfg.CertifyThreshold / int(defaultCnt)
- b := generateBlock(t, tc.db)
- tc.mClk.EXPECT().GetCurrentLayer().Return(b.LayerIndex).AnyTimes()
- tc.mb.EXPECT().GetBeacon(b.LayerIndex.GetEpoch()).Return(types.RandomBeacon(), nil).AnyTimes()
- require.NoError(t, tc.RegisterForCert(context.TODO(), b.LayerIndex, b.ID()))
- tc.mtortoise.EXPECT().OnHareOutput(b.LayerIndex, b.ID())
+ tt := []struct {
+ name string
+ concurrent bool
+ }{
+ {
+ name: "sequential",
+ },
+ {
+ name: "concurrent",
+ concurrent: true,
+ },
+ }
+ for _, tc := range tt {
+ tc := tc
+ t.Run(tc.name, func(t *testing.T) {
+ tcc := newTestCertifier(t)
+ numMsgs := tcc.cfg.CommitteeSize
+ cutoff := tcc.cfg.CertifyThreshold / int(defaultCnt)
+ b := generateBlock(t, tcc.db)
+ tcc.mClk.EXPECT().GetCurrentLayer().Return(b.LayerIndex).AnyTimes()
+ tcc.mb.EXPECT().GetBeacon(b.LayerIndex.GetEpoch()).Return(types.RandomBeacon(), nil).AnyTimes()
+ require.NoError(t, tcc.RegisterForCert(context.TODO(), b.LayerIndex, b.ID()))
+ if tc.concurrent {
+ tcc.mOracle.EXPECT().Validate(gomock.Any(), b.LayerIndex, eligibility.CertifyRound, tcc.cfg.CommitteeSize, gomock.Any(), gomock.Any(), defaultCnt).
+ Return(true, nil).MinTimes(cutoff).MaxTimes(numMsgs)
+ }
+ tcc.mtortoise.EXPECT().OnHareOutput(b.LayerIndex, b.ID())
+ var wg sync.WaitGroup
for i := 0; i < numMsgs; i++ {
nid, msg, encoded := genEncodedMsg(t, b.LayerIndex, b.ID())
- if i < cutoff {
- tc.mOracle.EXPECT().Validate(gomock.Any(), b.LayerIndex, eligibility.CertifyRound, tc.cfg.CommitteeSize, nid, msg.Proof, defaultCnt).
+ if !tc.concurrent && i < cutoff { // we know exactly which msgs will be validated
+ tcc.mOracle.EXPECT().Validate(gomock.Any(), b.LayerIndex, eligibility.CertifyRound, tcc.cfg.CommitteeSize, nid, msg.Proof, defaultCnt).
Return(true, nil)
}
- res := tc.HandleCertifyMessage(context.TODO(), "peer", encoded)
+ if tc.concurrent {
+ wg.Add(1)
+ go func(data []byte) {
+ res := tcc.HandleCertifyMessage(context.TODO(), "peer", data)
+ require.Equal(t, pubsub.ValidationAccept, res)
+ wg.Done()
+ }(encoded)
+ } else {
+ res := tcc.HandleCertifyMessage(context.TODO(), "peer", encoded)
require.Equal(t, pubsub.ValidationAccept, res)
}
- verifySaved(t, tc.db, b.LayerIndex, b.ID(), cutoff)
+ }
+ wg.Wait()
+ verifySaved(t, tcc.db, b.LayerIndex, b.ID(), cutoff)
+ })
+ }
}
func Test_HandleCertifyMessage_NotRegistered(t *testing.T) {

original_message:
block certifier: fix certifier concurrent map access (#3601)

## Motivation

race condition, added test that failed with -race -count 5 and pass after the fix


message:
cleanup: remove unused await sync
## Motivation
remove unused code

diff:
@@ -88,10 +88,6 @@ type Syncer struct {
lastLayerSynced atomic.Value
lastATXsSynced atomic.Value
- // awaitSyncedCh is the list of subscribers' channels to notify when this node enters synced state
- awaitSyncedCh []chan struct{}
- muSyncedCh sync.Mutex // protects the slice above from concurrent access
-
// awaitATXSyncedCh is the list of subscribers' channels to notify when this node enters ATX synced state
awaitATXSyncedCh []chan struct{}
muATXSyncedCh sync.Mutex // protects the slice above from concurrent access
@@ -130,7 +126,6 @@ func NewSyncer(
patrol: patrol,
syncTimer: time.NewTicker(conf.SyncInterval),
validateTimer: time.NewTicker(conf.SyncInterval * 3),
- awaitSyncedCh: make([]chan struct{}, 0),
awaitATXSyncedCh: make([]chan struct{}, 0),
shutdownCtx: shutdownCtx,
cancelFunc: cancel,
@@ -154,19 +149,6 @@ func (s *Syncer) Close() {
s.logger.With().Info("all syncer goroutines finished", log.Err(err))
}
-// RegisterForSynced returns a channel for notification when the node enters synced state.
-func (s *Syncer) RegisterForSynced(ctx context.Context) chan struct{} {
- ch := make(chan struct{})
- if s.IsSynced(ctx) {
- close(ch)
- return ch
- }
- s.muSyncedCh.Lock()
- defer s.muSyncedCh.Unlock()
- s.awaitSyncedCh = append(s.awaitSyncedCh, ch)
- return ch
-}
-
// RegisterForATXSynced returns a channel for notification when the node enters ATX synced state.
func (s *Syncer) RegisterForATXSynced() chan struct{} {
ch := make(chan struct{})
@@ -277,13 +259,6 @@ func (s *Syncer) setSyncState(ctx context.Context, newState syncState) {
if newState != synced {
return
}
- // notify subscribes
- s.muSyncedCh.Lock()
- defer s.muSyncedCh.Unlock()
- for _, ch := range s.awaitSyncedCh {
- close(ch)
- }
- s.awaitSyncedCh = make([]chan struct{}, 0)
}
}

diff:
@@ -130,9 +130,6 @@ func TestStartAndShutdown(t *testing.T) {
ctx, cancel := context.WithCancel(context.TODO())
ts := newTestSyncer(ctx, t, time.Millisecond*5)
- syncedCh := ts.syncer.RegisterForSynced(context.TODO())
- atxSynchedCh := ts.syncer.RegisterForATXSynced()
-
require.False(t, ts.syncer.IsSynced(context.TODO()))
require.False(t, ts.syncer.ListenToATXGossip())
require.False(t, ts.syncer.ListenToGossip())
@@ -140,21 +137,9 @@ func TestStartAndShutdown(t *testing.T) {
// the node is synced when current layer is <= 1
ts.syncer.Start(context.TODO())
- select {
- case <-syncedCh:
- case <-time.After(1 * time.Second):
- require.Fail(t, "node should be synced")
- }
-
- select {
- case <-atxSynchedCh:
- case <-time.After(1 * time.Second):
- require.Fail(t, "node should be atx synced")
- }
-
- require.True(t, ts.syncer.IsSynced(context.TODO()))
- require.True(t, ts.syncer.ListenToATXGossip())
- require.True(t, ts.syncer.ListenToGossip())
+ require.Eventually(t, func() bool {
+ return ts.syncer.ListenToATXGossip() && ts.syncer.ListenToGossip() && ts.syncer.IsSynced(context.TODO())
+ }, time.Second, 10*time.Millisecond)
cancel()
require.True(t, ts.syncer.isClosed())
@@ -213,6 +198,18 @@ func TestSynchronize_AllGood(t *testing.T) {
ts.mLyrFetcher.EXPECT().PollLayerData(gomock.Any(), lid).Return(okCh())
}
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ atxSyncedCh := ts.syncer.RegisterForATXSynced()
+ select {
+ case <-atxSyncedCh:
+ wg.Done()
+ case <-time.After(1 * time.Second):
+ require.Fail(t, "node should be atx synced")
+ }
+ }()
+
require.True(t, ts.syncer.synchronize(context.TODO()))
require.Equal(t, current.Sub(1), ts.syncer.getLastSyncedLayer())
require.Equal(t, current.GetEpoch(), ts.syncer.getLastSyncedATXs())
@@ -220,6 +217,18 @@ func TestSynchronize_AllGood(t *testing.T) {
require.True(t, ts.syncer.ListenToATXGossip())
require.True(t, ts.syncer.ListenToGossip())
require.False(t, ts.syncer.IsSynced(context.TODO()))
+
+ wg.Add(1)
+ go func() {
+ atxSyncedCh := ts.syncer.RegisterForATXSynced()
+ select {
+ case <-atxSyncedCh:
+ wg.Done()
+ case <-time.After(1 * time.Second):
+ require.Fail(t, "node should be atx synced")
+ }
+ }()
+ wg.Wait()
}
func TestSynchronize_FetchLayerDataFailed(t *testing.T) {
@@ -239,20 +248,6 @@ func TestSynchronize_FetchLayerDataFailed(t *testing.T) {
require.True(t, ts.syncer.ListenToATXGossip())
require.False(t, ts.syncer.ListenToGossip())
require.False(t, ts.syncer.IsSynced(context.TODO()))
-
- syncedCh := ts.syncer.RegisterForSynced(context.TODO())
- select {
- case <-syncedCh:
- require.Fail(t, "node should not be synced")
- case <-time.After(1 * time.Millisecond):
- }
-
- atxSyncedCh := ts.syncer.RegisterForATXSynced()
- select {
- case <-atxSyncedCh:
- case <-time.After(1 * time.Second):
- require.Fail(t, "node should be atx synced")
- }
}
func TestSynchronize_FailedInitialATXsSync(t *testing.T) {
@@ -265,6 +260,18 @@ func TestSynchronize_FailedInitialATXsSync(t *testing.T) {
}
ts.mLyrFetcher.EXPECT().GetEpochATXs(gomock.Any(), failedEpoch).Return(errors.New("no ATXs. should fail sync"))
+ var wg sync.WaitGroup
+ wg.Add(1)
+ go func() {
+ atxSyncedCh := ts.syncer.RegisterForATXSynced()
+ select {
+ case <-atxSyncedCh:
+ require.Fail(t, "node should not be atx synced")
+ case <-time.After(100 * time.Millisecond):
+ wg.Done()
+ }
+ }()
+
require.False(t, ts.syncer.synchronize(context.TODO()))
require.Equal(t, types.GetEffectiveGenesis(), ts.syncer.getLastSyncedLayer())
require.Equal(t, failedEpoch-1, ts.syncer.getLastSyncedATXs())
@@ -273,19 +280,17 @@ func TestSynchronize_FailedInitialATXsSync(t *testing.T) {
require.False(t, ts.syncer.ListenToGossip())
require.False(t, ts.syncer.IsSynced(context.TODO()))
- syncedCh := ts.syncer.RegisterForSynced(context.TODO())
- select {
- case <-syncedCh:
- require.Fail(t, "node should not be synced")
- case <-time.After(100 * time.Millisecond):
- }
-
+ wg.Add(1)
+ go func() {
atxSyncedCh := ts.syncer.RegisterForATXSynced()
select {
case <-atxSyncedCh:
require.Fail(t, "node should not be atx synced")
case <-time.After(100 * time.Millisecond):
+ wg.Done()
}
+ }()
+ wg.Wait()
}
func startWithSyncedState(t *testing.T, ts *testSyncer) types.LayerID {
@@ -442,23 +447,9 @@ func TestFromSyncedToNotSynced(t *testing.T) {
ts := newSyncerWithoutSyncTimer(t)
ts.mLyrFetcher.EXPECT().GetEpochATXs(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
- syncedCh := ts.syncer.RegisterForSynced(context.TODO())
- atxSyncedCh := ts.syncer.RegisterForATXSynced()
require.True(t, ts.syncer.synchronize(context.TODO()))
require.True(t, ts.syncer.IsSynced(context.TODO()))
- select {
- case <-syncedCh:
- case <-time.After(1 * time.Second):
- require.Fail(t, "node should be synced")
- }
-
- select {
- case <-atxSyncedCh:
- case <-time.After(1 * time.Second):
- require.Fail(t, "node should be atx synced")
- }
-
// cause the syncer to get out of synced and then wait again
lyr := types.GetEffectiveGenesis().Add(1)
current := ts.msh.LatestLayer().Add(outOfSyncThreshold)
@@ -511,8 +502,6 @@ func waitOutGossipSync(t *testing.T, current types.LayerID, ts *testSyncer) {
require.False(t, ts.syncer.IsSynced(context.TODO()))
// done one full layer of gossip sync, now it is synced
- syncedCh := ts.syncer.RegisterForSynced(context.TODO())
- atxSyncedCh := ts.syncer.RegisterForATXSynced()
lyr = lyr.Add(1)
current = current.Add(1)
ts.mTicker.advanceToLayer(current)
@@ -522,19 +511,6 @@ func waitOutGossipSync(t *testing.T, current types.LayerID, ts *testSyncer) {
return okCh()
})
require.True(t, ts.syncer.synchronize(context.TODO()))
-
- select {
- case <-syncedCh:
- case <-time.After(1 * time.Second):
- require.Fail(t, "node should be synced")
- }
-
- select {
- case <-atxSyncedCh:
- case <-time.After(1 * time.Second):
- require.Fail(t, "node should be atx synced")
- }
-
require.True(t, ts.syncer.dataSynced())
require.True(t, ts.syncer.ListenToATXGossip())
require.True(t, ts.syncer.ListenToGossip())
@@ -634,27 +610,6 @@ func TestSync_AlsoSyncProcessedLayer(t *testing.T) {
require.Equal(t, lyr, ts.syncer.getLastSyncedLayer())
}
-func TestSyncer_setSyncedTwice_NoError(t *testing.T) {
- ts := newSyncerWithoutSyncTimer(t)
-
- sync := ts.syncer.RegisterForSynced(context.TODO())
- select {
- case <-sync:
- require.Fail(t, "should not have reached synced state yet")
- case <-time.After(100 * time.Millisecond):
- }
-
- ts.syncer.setSyncState(context.TODO(), synced)
-
- select {
- case <-sync:
- case <-time.After(1 * time.Second):
- require.Fail(t, "should have reached synced state")
- }
-
- require.NotPanics(t, func() { ts.syncer.setSyncState(context.TODO(), synced) })
-}
-
func TestSyncer_setATXSyncedTwice_NoError(t *testing.T) {
ts := newSyncerWithoutSyncTimer(t)

original_message:
cleanup: remove unused await sync (#3618)

## Motivation

remove unused code


message:
partition test: increase healing time
## Motivation
remove flakiness in the systest

diff:
@@ -16,6 +16,10 @@ env:
TD_QUEUE_ZONE: ${{ secrets.TD_QUEUE_ZONE }}
DUMP_QUEUE_NAME: ${{ secrets.DUMP_QUEUE_NAME }}
DUMP_QUEUE_ZONE: ${{ secrets.DUMP_QUEUE_ZONE }}
+ CI_CLUSTER_NAME: ${{ secrets.CI_CLUSTER_NAME }}
+ CI_GCP_CREDENTIALS: ${{ secrets.CI_GCP_CREDENTIALS }}
+ CI_GCP_PROJECT_ID: ${{ secrets.CI_GCP_PROJECT_ID }}
+ CI_REGION_NAME: ${{ secrets.CI_REGION_NAME }}
# Trigger the workflow on all pull requests, and on push to specific branches
on:
@@ -279,13 +283,13 @@ jobs:
uses: "google-github-actions/auth@v1"
with:
# GCP_CREDENTIALS is minified JSON of service account
- credentials_json: "${{ secrets.GCP_CREDENTIALS }}"
+ credentials_json: "${{ secrets.CI_GCP_CREDENTIALS }}"
- name: Configure gcloud
uses: "google-github-actions/setup-gcloud@v1"
- name: Configure kubectl
- run: gcloud container clusters get-credentials ${{ secrets.CI_CLUSTER_NAME }} --zone ${{ secrets.CI_ZONE_NAME }} --project ${{ secrets.GCP_PROJECT_ID }}
+ run: gcloud container clusters get-credentials ${{ secrets.CI_CLUSTER_NAME }} --region ${{ secrets.CI_REGION_NAME }} --project ${{ secrets.CI_GCP_PROJECT_ID }}
- name: Login to Docker Hub
uses: docker/login-action@v2
@@ -313,7 +317,7 @@ jobs:
go-version: ${{ env.go-version }}
- name: Run tests I
- timeout-minutes: 70
+ timeout-minutes: 55
env:
test_id: systest-${{ steps.vars.outputs.sha_short }}
label: sanity
@@ -326,7 +330,7 @@ jobs:
run: make -C systest run test_name=.
- name: Run tests II
- timeout-minutes: 45
+ timeout-minutes: 30
env:
test_id: systest-${{ steps.vars.outputs.sha_short }}
label: destructive

diff:
@@ -169,7 +169,8 @@ func TestPartition_30_70(t *testing.T) {
tctx := testcontext.New(t, testcontext.Labels("destructive"))
cl, err := cluster.Reuse(tctx, cluster.WithKeys(10))
require.NoError(t, err)
- testPartition(t, tctx, cl, 30, 2)
+ // TODO: re-assess the number of epoch required for healing.
+ testPartition(t, tctx, cl, 30, 4)
}
func TestPartition_50_50(t *testing.T) {
@@ -178,5 +179,6 @@ func TestPartition_50_50(t *testing.T) {
tctx := testcontext.New(t, testcontext.Labels("destructive"))
cl, err := cluster.Reuse(tctx, cluster.WithKeys(10))
require.NoError(t, err)
- testPartition(t, tctx, cl, 50, 3)
+ // TODO: re-assess the number of epoch required for healing.
+ testPartition(t, tctx, cl, 50, 5)
}

original_message:
partition test: increase healing time (#3882)

## Motivation
remove flakiness in the systest

Co-authored-by: andres-spacemesh <106999199+andres-spacemesh@users.noreply.github.com>


message:
systest: fail fast tests to avoid timeout
## Motivation
attempt to differentiate test failures vs test timeout
make systest I log at info level

diff:
@@ -325,7 +325,7 @@ jobs:
node_selector: cloud.google.com/gke-nodepool=gha
size: 50
bootstrap: 10m
- level: debug
+ level: info
clusters: 2
run: make -C systest run test_name=.
@@ -338,7 +338,7 @@ jobs:
node_selector: cloud.google.com/gke-nodepool=gha
size: 30
bootstrap: 10m
- level: debug
+ level: info
clusters: 2
run: make -C systest run test_name=.

old_path: systest/Dockerfile

new_path: systest/Dockerfile

diff:
@@ -7,7 +7,7 @@ COPY go.sum .
RUN go mod download
COPY . .
-RUN --mount=type=cache,target=/root/.cache/go-build go test -v -c -o /build/tests.test ./systest/tests/
+RUN --mount=type=cache,target=/root/.cache/go-build go test -failfast -v -c -o /build/tests.test ./systest/tests/
FROM alpine
COPY --from=build /build/tests.test /bin/tests

original_message:
systest: fail fast tests to avoid timeout (#3892)

## Motivation
- attempt to differentiate test failures vs test timeout
- make systest I log at info level


message:
address review feedback for PR 3960
## Motivation
<!-- Please mention the issue fixed by this PR or detailed motivation -->
review feedback for

diff:
@@ -297,7 +297,7 @@ func (h *Handler) StoreAtx(ctx context.Context, atx *types.VerifiedActivationTx)
malicious, err := h.cdb.IsMalicious(atx.NodeID())
if err != nil {
- return fmt.Errorf("store atx: %w", err)
+ return fmt.Errorf("checking if node is malicious: %w", err)
}
var proof *types.MalfeasanceProof
h.cdb.WithTx(ctx, func(dbtx *sql.Tx) error {
@@ -322,7 +322,7 @@ func (h *Handler) StoreAtx(ctx context.Context, atx *types.VerifiedActivationTx)
},
}
if err = h.cdb.AddMalfeasanceProof(atx.NodeID(), proof, dbtx); err != nil {
- return err
+ return fmt.Errorf("adding malfeasense proof: %w", err)
}
h.log.With().Warning("smesher produced more than one atx in the same epoch",
log.Stringer("smesher", atx.NodeID()),

diff:
@@ -21,9 +21,9 @@ type MeshService struct {
genTime api.GenesisTimeAPI
layersPerEpoch uint32
genesisID types.Hash20
- layerDurationSec int
- layerAvgSize int
- txsPerProposal int
+ layerDurationSec uint32
+ layerAvgSize uint32
+ txsPerProposal uint32
}
// RegisterService registers this service with a grpc server instance.
@@ -34,8 +34,8 @@ func (s MeshService) RegisterService(server *Server) {
// NewMeshService creates a new service using config data.
func NewMeshService(
msh api.MeshAPI, cstate api.ConservativeState, genTime api.GenesisTimeAPI,
- layersPerEpoch uint32, genesisID types.Hash20, layerDurationSec int,
- layerAvgSize int, txsPerProposal int,
+ layersPerEpoch uint32, genesisID types.Hash20, layerDurationSec uint32,
+ layerAvgSize uint32, txsPerProposal uint32,
) *MeshService {
return &MeshService{
mesh: msh,

diff:
@@ -363,10 +363,10 @@ func (app *App) Initialize() (err error) {
// hare's max time duration to run consensus for a layer
maxHareRoundsPerLayer := 1 + app.Config.HARE.LimitIterations*hare.RoundsPerIteration // pre-round + 4 rounds per iteration
maxHareLayerDurationSec := app.Config.HARE.WakeupDelta + maxHareRoundsPerLayer*app.Config.HARE.RoundDuration
- if app.Config.LayerDurationSec*int(app.Config.Tortoise.Zdist) <= maxHareLayerDurationSec {
+ if app.Config.LayerDurationSec*app.Config.Tortoise.Zdist <= uint32(maxHareLayerDurationSec) {
log.With().Error("incompatible params",
log.Uint32("tortoise_zdist", app.Config.Tortoise.Zdist),
- log.Int("layer_duration", app.Config.LayerDurationSec),
+ log.Uint32("layer_duration", app.Config.LayerDurationSec),
log.Int("hare_wakeup_delta", app.Config.HARE.WakeupDelta),
log.Int("hare_limit_iterations", app.Config.HARE.LimitIterations),
log.Int("hare_round_duration", app.Config.HARE.RoundDuration))
@@ -813,7 +813,7 @@ func (app *App) startAPIServices(ctx context.Context) {
registerService(grpcserver.NewGlobalStateService(app.mesh, app.conState))
}
if apiConf.StartMeshService {
- registerService(grpcserver.NewMeshService(app.mesh, app.conState, app.clock, app.Config.LayersPerEpoch, app.Config.Genesis.GenesisID(), layerDuration, app.Config.LayerAvgSize, app.Config.TxsPerProposal))
+ registerService(grpcserver.NewMeshService(app.mesh, app.conState, app.clock, app.Config.LayersPerEpoch, app.Config.Genesis.GenesisID(), layerDuration, app.Config.LayerAvgSize, uint32(app.Config.TxsPerProposal)))
}
if apiConf.StartNodeService {
nodeService := grpcserver.NewNodeService(app.host, app.mesh, app.clock, app.syncer, app.atxBuilder)

diff:
@@ -44,9 +44,9 @@ func AddCommands(cmd *cobra.Command) {
cfg.Genesis.GenesisTime, "Time of the genesis layer in 2019-13-02T17:02:00+00:00 format")
cmd.PersistentFlags().StringVar(&cfg.Genesis.ExtraData, "genesis-extra-data",
cfg.Genesis.ExtraData, "genesis extra-data will be committed to the genesis id")
- cmd.PersistentFlags().IntVar(&cfg.LayerDurationSec, "layer-duration-sec",
+ cmd.PersistentFlags().Uint32Var(&cfg.LayerDurationSec, "layer-duration-sec",
cfg.LayerDurationSec, "Duration between layers in seconds")
- cmd.PersistentFlags().IntVar(&cfg.LayerAvgSize, "layer-average-size",
+ cmd.PersistentFlags().Uint32Var(&cfg.LayerAvgSize, "layer-average-size",
cfg.LayerAvgSize, "Layer Avg size")
cmd.PersistentFlags().BoolVar(&cfg.PprofHTTPServer, "pprof-server",
cfg.PprofHTTPServer, "enable http pprof server")

diff:
@@ -83,8 +83,8 @@ type BaseConfig struct {
OracleServer string `mapstructure:"oracle_server"`
OracleServerWorldID int `mapstructure:"oracle_server_worldid"`
- LayerDurationSec int `mapstructure:"layer-duration-sec"`
- LayerAvgSize int `mapstructure:"layer-average-size"`
+ LayerDurationSec uint32 `mapstructure:"layer-duration-sec"`
+ LayerAvgSize uint32 `mapstructure:"layer-average-size"`
LayersPerEpoch uint32 `mapstructure:"layers-per-epoch"`
PoETServers []string `mapstructure:"poet-server"`

diff:
@@ -51,7 +51,7 @@ func (db *CachedDB) MalfeasanceCacheSize() int {
// IsMalicious returns true if the NodeID is malicious.
func (db *CachedDB) IsMalicious(id types.NodeID) (bool, error) {
if id == types.EmptyNodeID {
- return false, errors.New("cache db: get empty node id")
+ log.Fatal("invalid argument to IsMalicious")
}
db.mu.Lock()
@@ -77,7 +77,7 @@ func (db *CachedDB) IsMalicious(id types.NodeID) (bool, error) {
// GetMalfeasanceProof gets the malfeasance proof associated with the NodeID.
func (db *CachedDB) GetMalfeasanceProof(id types.NodeID) (*types.MalfeasanceProof, error) {
if id == types.EmptyNodeID {
- return nil, errors.New("cache db: get empty node id")
+ log.Fatal("invalid argument to GetMalfeasanceProof")
}
db.mu.Lock()
@@ -99,7 +99,7 @@ func (db *CachedDB) GetMalfeasanceProof(id types.NodeID) (*types.MalfeasanceProo
func (db *CachedDB) AddMalfeasanceProof(id types.NodeID, proof *types.MalfeasanceProof, dbtx *sql.Tx) error {
if id == types.EmptyNodeID {
- return errors.New("cache db: add empty node id")
+ log.Fatal("invalid argument to AddMalfeasanceProof")
}
encoded, err := codec.Encode(proof)

diff:
@@ -22,32 +22,6 @@ import (
"github.com/spacemeshos/go-spacemesh/sql/transactions"
)
-func TestMalfeasanceProof_EmptyNodeID(t *testing.T) {
- db := sql.InMemory()
- cdb := datastore.NewCachedDB(db, logtest.New(t))
- require.Equal(t, 0, cdb.MalfeasanceCacheSize())
-
- // empty node id
- got, err := cdb.GetMalfeasanceProof(types.NodeID{})
- require.Error(t, err)
- require.Nil(t, got)
-
- proof := &types.MalfeasanceProof{
- Layer: types.NewLayerID(11),
- Proof: types.Proof{
- Type: types.MultipleBallots,
- Data: &types.BallotProof{
- Messages: [2]types.BallotProofMsg{
- {},
- {},
- },
- },
- },
- }
- require.Error(t, cdb.AddMalfeasanceProof(types.NodeID{}, proof, nil))
- require.Equal(t, 0, cdb.MalfeasanceCacheSize())
-}
-
func TestMalfeasanceProof_Honest(t *testing.T) {
db := sql.InMemory()
cdb := datastore.NewCachedDB(db, logtest.New(t))
@@ -127,7 +101,6 @@ func TestMalfeasanceProof_Dishonest(t *testing.T) {
},
},
}
- require.Error(t, cdb.AddMalfeasanceProof(types.NodeID{}, proof, nil))
nodeID1 := types.NodeID{1}
require.NoError(t, cdb.AddMalfeasanceProof(nodeID1, proof, nil))

original_message:
address review feedback for PR 3960 (#3968)

## Motivation
<!-- Please mention the issue fixed by this PR or detailed motivation -->
review feedback for https://github.com/spacemeshos/go-spacemesh/pull/3960


message:
hare: use beacon as is in the vrf message
## Motivation
make hare encodes beacon consistently as other vrf messages.

diff:
@@ -386,7 +386,6 @@ func (pd *ProtocolDriver) GetBeacon(targetEpoch types.EpochID) (types.Beacon, er
}
if errors.Is(err, sql.ErrNotFound) {
- pd.logger.With().Warning("beacon not available", targetEpoch)
return types.EmptyBeacon, errBeaconNotCalculated
}
pd.logger.With().Error("failed to get beacon from db", targetEpoch, log.Err(err))

diff:
@@ -2,7 +2,6 @@ package eligibility
import (
"context"
- "encoding/binary"
"errors"
"fmt"
"math"
@@ -157,38 +156,19 @@ func New(
type VrfMessage struct {
Type types.EligibilityType
Nonce types.VRFPostIndex
- Beacon uint32
+ Beacon types.Beacon
Round uint32
Layer types.LayerID
}
-func encodeBeacon(beacon types.Beacon) uint32 {
- return binary.LittleEndian.Uint32(beacon.Bytes())
-}
-
-func (o *Oracle) getBeaconValue(ctx context.Context, epochID types.EpochID) (uint32, error) {
- beacon, err := o.beacons.GetBeacon(epochID)
- if err != nil {
- return 0, fmt.Errorf("get beacon: %w", err)
- }
-
- value := encodeBeacon(beacon)
- o.WithContext(ctx).With().Info("eligibility: beacon value for epoch",
- epochID,
- beacon,
- log.Uint32("beacon_val", value),
- )
- return value, nil
-}
-
// buildVRFMessage builds the VRF message used as input for the BLS (msg=Beacon##Layer##Round).
func (o *Oracle) buildVRFMessage(ctx context.Context, nonce types.VRFPostIndex, layer types.LayerID, round uint32) ([]byte, error) {
- v, err := o.getBeaconValue(ctx, layer.GetEpoch())
+ beacon, err := o.beacons.GetBeacon(layer.GetEpoch())
if err != nil {
return nil, fmt.Errorf("get beacon: %w", err)
}
- msg := VrfMessage{Type: types.EligibilityHare, Nonce: nonce, Beacon: v, Round: round, Layer: layer}
+ msg := VrfMessage{Type: types.EligibilityHare, Nonce: nonce, Beacon: beacon, Round: round, Layer: layer}
buf, err := codec.Encode(&msg)
if err != nil {
o.WithContext(ctx).With().Fatal("failed to encode", log.Err(err))
@@ -234,7 +214,7 @@ func calcVrfFrac(vrfSig []byte) fixed.Fixed {
func (o *Oracle) prepareEligibilityCheck(ctx context.Context, layer types.LayerID, round uint32, committeeSize int, id types.NodeID, nonce types.VRFPostIndex, vrfSig []byte) (n int, p, vrfFrac fixed.Fixed, done bool, err error) {
logger := o.WithContext(ctx).WithFields(
layer,
- log.FieldNamed("message_node_id", id),
+ log.Stringer("sender_id", id),
log.Uint32("round", round),
log.Int("committee_size", committeeSize),
)
@@ -253,7 +233,6 @@ func (o *Oracle) prepareEligibilityCheck(ctx context.Context, layer types.LayerI
// validate message
if !o.vrfVerifier.Verify(id, msg, vrfSig) {
logger.With().Info("eligibility: a node did not pass vrf signature verification",
- log.FieldNamed("sender_id", id),
log.FieldNamed("sender_vrf_nonce", nonce),
)
return 0, fixed.Fixed{}, fixed.Fixed{}, true, nil

diff:
@@ -24,7 +24,7 @@ func (t *VrfMessage) EncodeScale(enc *scale.Encoder) (total int, err error) {
total += n
}
{
- n, err := scale.EncodeCompact32(enc, uint32(t.Beacon))
+ n, err := scale.EncodeByteArray(enc, t.Beacon[:])
if err != nil {
return total, err
}
@@ -65,12 +65,11 @@ func (t *VrfMessage) DecodeScale(dec *scale.Decoder) (total int, err error) {
t.Nonce = types.VRFPostIndex(field)
}
{
- field, n, err := scale.DecodeCompact32(dec)
+ n, err := scale.DecodeByteArray(dec, t.Beacon[:])
if err != nil {
return total, err
}
total += n
- t.Beacon = uint32(field)
}
{
field, n, err := scale.DecodeCompact32(dec)

diff:
@@ -869,13 +869,6 @@ func TestMaxSupportedN(t *testing.T) {
})
}
-func TestEncodeBeacon(t *testing.T) {
- beacon := types.HexToBeacon("0xaeebad4a796fcc2e15dc4c6061b45ed9b373f26adfc798ca7d2d8cc58182718e")
- require.Len(t, beacon, 4)
- expected := uint32(0x4aadebae)
- require.Equal(t, expected, encodeBeacon(beacon))
-}
-
func FuzzVrfMessageConsistency(f *testing.F) {
tester.FuzzConsistency[VrfMessage](f)
}

original_message:
hare: use beacon as is in the vrf message (#4022)

## Motivation
make hare encodes beacon consistently as other vrf messages.


message:
hare: add log for round completion
## Motivation
<!-- Please mention the issue fixed by this PR or detailed motivation -->
log eligibility count upon hare round completion

diff:
@@ -859,6 +859,7 @@ func (proc *consensusProcess) processNotifyMsg(ctx context.Context, msg *Msg) {
log.String("current_set", proc.value.String()),
log.Uint32("current_round", proc.getRound()),
proc.layer,
+ log.Object("notify_count", notifyCount),
log.Int("set_size", proc.value.Size()))
proc.report(completed)
numIterations.Observe(float64(proc.getRound()))

diff:
@@ -110,6 +110,8 @@ func (ct *commitTracker) HasEnoughCommits() bool {
if ci.numDishonest > 0 {
ct.logger.With().Warning("counting votes from malicious identities",
log.Object("eligibility_count", ci))
+ } else {
+ ct.logger.With().Info("commit round completed", log.Object("eligibility_count", ci))
}
}
return enough

diff:
@@ -515,7 +515,8 @@ func (o *Oracle) actives(ctx context.Context, targetLayer types.LayerID) (map[ty
}
if len(hareActiveSet) > 0 {
- logger.With().Debug("successfully got hare active set for layer range",
+ logger.With().Info("successfully got hare active set for layer range",
+ log.Uint32("layer_range_epoch", uint32(safeLayerStart.GetEpoch())),
log.Int("count", len(hareActiveSet)))
o.activesCache.Add(safeLayerStart.GetEpoch(), hareActiveSet)
return hareActiveSet, nil

diff:
@@ -96,6 +96,11 @@ func (st *statusTracker) AnalyzeStatusMessages(isValid func(m *Msg) bool) {
})
st.tally = &ci
st.analyzed = true
+ if st.tally.Meet(st.threshold) {
+ st.logger.With().Info("status round completed", log.Object("eligibility_count", st.tally))
+ } else {
+ st.logger.With().Warning("status round failed", log.Object("eligibility_count", st.tally))
+ }
}
// IsSVPReady returns true if there are enough statuses to build an SVP, false otherwise.

original_message:
hare: add log for round completion (#4038)

## Motivation
<!-- Please mention the issue fixed by this PR or detailed motivation -->
log eligibility count upon hare round completion


message:
LayersQuery: populate layer hash with aggregated hash
## Motivation
<!-- Please mention the issue fixed by this PR or detailed motivation -->
removed the unused field for layer hash.
replace it with aggregated layer hash.

diff:
@@ -300,6 +300,10 @@ func (m *MeshAPIMock) GetATXs(context.Context, []types.ATXID) (map[types.ATXID]*
return atxs, nil
}
+func (m *MeshAPIMock) MeshHash(types.LayerID) (types.Hash32, error) {
+ return types.RandomHash(), nil
+}
+
type ConStateAPIMock struct {
returnTx map[types.TransactionID]*types.Transaction
layerApplied map[types.TransactionID]*types.LayerID
@@ -1637,6 +1641,7 @@ func TestMeshService(t *testing.T) {
resLayerNine := res.Layer[9]
require.Equal(t, uint32(9), resLayerNine.Number.Number, "layer nine is ninth")
+ require.NotEmpty(t, resLayerNine.Hash)
require.Equal(t, pb.Layer_LAYER_STATUS_UNSPECIFIED, resLayerNine.Status, "later layer is unconfirmed")
},
},
@@ -2290,6 +2295,7 @@ func TestLayerStream_comprehensive(t *testing.T) {
require.NoError(t, err, "got error from stream")
require.Equal(t, uint32(0), res.Layer.Number.Number)
require.Equal(t, events.LayerStatusTypeConfirmed, int(res.Layer.Status))
+ require.NotEmpty(t, res.Layer.Hash)
checkLayer(t, res.Layer)
}

diff:
@@ -356,11 +356,19 @@ func (s MeshService) readLayer(ctx context.Context, layerID types.LayerID, layer
log.With().Debug("no state root for layer",
layer, log.String("status", layerStatus.String()), log.Err(err))
}
+ hash, err := s.mesh.MeshHash(layerID)
+ if err != nil {
+ // This is expected. We can only retrieve state root for a layer that was applied to state,
+ // which only happens after it's approved/confirmed.
+ log.With().Debug("no mesh hash at layer",
+ layer, log.String("status", layerStatus.String()), log.Err(err))
+ }
return &pb.Layer{
Number: &pb.LayerNumber{Number: layer.Index().Uint32()},
Status: layerStatus,
Blocks: blocks,
Activations: pbActivations,
+ Hash: hash.Bytes(),
RootStateHash: stateRoot.Bytes(),
}, nil
}

diff:
@@ -55,6 +55,7 @@ type MeshAPI interface {
LatestLayer() types.LayerID
LatestLayerInState() types.LayerID
ProcessedLayer() types.LayerID
+ MeshHash(types.LayerID) (types.Hash32, error)
}
// NOTE that mockgen doesn't use source-mode to avoid generating mocks for all interfaces in this file.

diff:
@@ -134,6 +134,11 @@ func (msh *Mesh) LatestLayer() types.LayerID {
return msh.latestLayer.Load().(types.LayerID)
}
+// MeshHash returns the aggregated mesh hash at the specified layer.
+func (msh *Mesh) MeshHash(lid types.LayerID) (types.Hash32, error) {
+ return layers.GetAggregatedHash(msh.cdb, lid)
+}
+
// MissingLayer is a layer in (latestLayerInState, processLayer].
// this layer is missing critical data (valid blocks or transactions)
// and can't be applied to the state.

original_message:
LayersQuery: populate layer hash with aggregated hash (#4046)

## Motivation
<!-- Please mention the issue fixed by this PR or detailed motivation -->
#4026 removed the unused field for layer hash.
replace it with aggregated layer hash.


message: fix bug due to synctex

diff:
@@ -64,7 +64,12 @@ export class Locator {
}
const configuration = vscode.workspace.getConfiguration('latex-workshop')
const pdfFile = this.extension.manager.tex2pdf(this.extension.manager.rootFile)
- const args = ['view', '-i', `${position.line + 1}:${position.character + 1}:${filePath}`, "-o", pdfFile]
+ let line = position.line + 1
+ if (vscode.window.activeTextEditor.document.lineCount === line &&
+ vscode.window.activeTextEditor.document.lineAt(line - 1).text === '') {
+ line -= 1
+ }
+ const args = ['view', '-i', `${line}:${position.character + 1}:${filePath}`, '-o', pdfFile]
this.extension.logger.addLogMessage(`Executing synctex with args ${args}`)
const proc = cp.spawn(configuration.get('synctex.path') as string, args)

repo: james-yu/latex-workshop

original_message: fix bug due to synctex


message: support .chktexrc

diff:
import * as vscode from 'vscode'
import * as path from 'path'
+import * as fs from 'fs'
import {ChildProcess, spawn, SpawnOptions} from 'child_process'
import {EOL} from 'os'
@@ -25,6 +26,12 @@ export class Linter {
const configuration = vscode.workspace.getConfiguration('latex-workshop')
const command = configuration.get('chktex.path') as string
const args = configuration.get('chktex.args.active') as string[]
+ if (this.extension.manager.rootDir) {
+ const rcPath = path.join(this.extension.manager.rootDir, '.chktexrc')
+ if (fs.existsSync(rcPath)) {
+ args.push('-l', rcPath)
+ }
+ }
const requiredArgs = ['-I0', '-f%f:%l:%c:%d:%k:%n:%m\n']
let stdout: string
@@ -49,6 +56,12 @@ export class Linter {
const configuration = vscode.workspace.getConfiguration('latex-workshop')
const command = configuration.get('chktex.path') as string
const args = configuration.get('chktex.args.root') as string[]
+ if (this.extension.manager.rootDir) {
+ const rcPath = path.join(this.extension.manager.rootDir, '.chktexrc')
+ if (fs.existsSync(rcPath)) {
+ args.push('-l', rcPath)
+ }
+ }
const requiredArgs = ['-f%f:%l:%c:%d:%k:%n:%m\n', '%DOC%'.replace('%DOC%', filePath)]
let stdout: string

original_message: support .chktexrc


message: ignore empty bib warn

diff:
@@ -8,6 +8,7 @@ const latexFatalPattern = /Fatal error occurred, no output PDF file produced!/gm
const latexError = /^(?:(.*):(\d+):|!)(?: (.+) Error:)? (.+?)\.?$/
const latexBox = /^((?:Over|Under)full \\[vh]box \([^)]*\)) in paragraph at lines (\d+)--(\d+)$/
const latexWarn = /^((?:(?:Class|Package) \S+)|LaTeX) (Warning|Info):\s+(.*?)(?: on input line (\d+))?\.$/
+const bibEmpty = /^Empty `thebibliography' environment/
const latexmkPattern = /^Latexmk:\sapplying\srule/gm
const latexmkLog = /^Latexmk:\sapplying\srule/
@@ -159,7 +160,7 @@ export class Parser {
}
}
// push final result
- if (currentResult.type !== '') {
+ if (currentResult.type !== '' && !currentResult.text.match(bibEmpty)) {
this.buildLog.push(currentResult)
}
this.extension.logger.addLogMessage(`LaTeX log parsed with ${this.buildLog.length} messages.`)

original_message: ignore empty bib warn


message: latex inline math completion

diff:
"snippet":"begin{${1:env}}\n\t$2\n\\\\end{${1:env}}",
"detail":"begin a new environment"
},
+ "inlinemath":{
+ "command":"(",
+ "snippet":"(${1}\\)${0}",
+ "detail":"inline math \\( ... \\)"
+ },
"title":{
"command":"title",
"snippet":"title{${1:text}}"
"pagebreak":{
"command":"pagebreak"
},
- " ":{
- "command":" Press ENTER to insert a new line.",
- "snippet":"\n"
- },
"noindent":{
"command":"noindent"
},

diff:
@@ -209,7 +209,7 @@ export async function activate(context: vscode.ExtensionContext) {
context.subscriptions.push(vscode.languages.registerDocumentSymbolProvider('latex', new DocSymbolProvider(extension)))
context.subscriptions.push(vscode.languages.registerWorkspaceSymbolProvider(new ProjectSymbolProvider(extension)))
context.subscriptions.push(vscode.languages.registerCompletionItemProvider('tex', extension.completer, '\\', '{'))
- context.subscriptions.push(vscode.languages.registerCompletionItemProvider('latex', extension.completer, '\\', '{', ','))
+ context.subscriptions.push(vscode.languages.registerCompletionItemProvider('latex', extension.completer, '\\', '{', ',', '('))
context.subscriptions.push(vscode.languages.registerCodeActionsProvider('latex', extension.codeActions))
context.subscriptions.push(vscode.window.registerTreeDataProvider('latex-outline', extension.nodeProvider))

diff:
@@ -48,6 +48,21 @@ export class Completer implements vscode.CompletionItemProvider {
provideCompletionItems(document: vscode.TextDocument, position: vscode.Position, _token: vscode.CancellationToken) : Promise<vscode.CompletionItem[]> {
return new Promise((resolve, _reject) => {
+ if (document.lineAt(position.line).text[position.character - 1] === '(') {
+ if (position.character > 1 && document.lineAt(position.line).text[position.character - 2] === '\\') {
+ const inlineMath = new vscode.CompletionItem('\\(', vscode.CompletionItemKind.Function)
+ inlineMath.insertText = new vscode.SnippetString('${1}\\)${0}')
+ inlineMath.detail = 'inline math \\( ... \\)'
+ if (vscode.workspace.getConfiguration('editor', document.uri).get('autoClosingBrackets')) {
+ inlineMath.range = new vscode.Range(position, position.translate(0, 1))
+ }
+ resolve([inlineMath])
+ return
+ } else {
+ return
+ }
+ }
+
const line = document.lineAt(position.line).text.substr(0, position.character)
for (const type of ['citation', 'reference', 'environment', 'command']) {
const suggestions = this.completion(type, line)

original_message: latex inline math completion


message: add latex displaymath snippet

diff:
"snippet":"begin{${1:env}}\n\t$2\n\\\\end{${1:env}}",
"detail":"begin a new environment"
},
- "inlinemath":{
+ "latexinlinemath":{
"command":"(",
"snippet":"(${1}\\)${0}",
"detail":"inline math \\( ... \\)"
},
+ "latexdisplaymath":{
+ "command":"[",
+ "snippet":"[${1}\\]${0}",
+ "detail":"display math \\[ ... \\]"
+ },
"title":{
"command":"title",
"snippet":"title{${1:text}}"

diff:
@@ -209,7 +209,7 @@ export async function activate(context: vscode.ExtensionContext) {
context.subscriptions.push(vscode.languages.registerDocumentSymbolProvider('latex', new DocSymbolProvider(extension)))
context.subscriptions.push(vscode.languages.registerWorkspaceSymbolProvider(new ProjectSymbolProvider(extension)))
context.subscriptions.push(vscode.languages.registerCompletionItemProvider('tex', extension.completer, '\\', '{'))
- context.subscriptions.push(vscode.languages.registerCompletionItemProvider('latex', extension.completer, '\\', '{', ',', '('))
+ context.subscriptions.push(vscode.languages.registerCompletionItemProvider('latex', extension.completer, '\\', '{', ',', '(', '['))
context.subscriptions.push(vscode.languages.registerCodeActionsProvider('latex', extension.codeActions))
context.subscriptions.push(vscode.window.registerTreeDataProvider('latex-outline', extension.nodeProvider))

diff:
@@ -48,15 +48,23 @@ export class Completer implements vscode.CompletionItemProvider {
provideCompletionItems(document: vscode.TextDocument, position: vscode.Position, _token: vscode.CancellationToken) : Promise<vscode.CompletionItem[]> {
return new Promise((resolve, _reject) => {
- if (document.lineAt(position.line).text[position.character - 1] === '(') {
+ const invokeChar = document.lineAt(position.line).text[position.character - 1]
+ if (['(', '['].indexOf(invokeChar) > -1) {
if (position.character > 1 && document.lineAt(position.line).text[position.character - 2] === '\\') {
- const inlineMath = new vscode.CompletionItem('\\(', vscode.CompletionItemKind.Function)
- inlineMath.insertText = new vscode.SnippetString('${1}\\)${0}')
- inlineMath.detail = 'inline math \\( ... \\)'
+ let mathSnippet
+ if (invokeChar === '(') {
+ mathSnippet = new vscode.CompletionItem('\\(', vscode.CompletionItemKind.Function)
+ mathSnippet.insertText = new vscode.SnippetString('${1}\\)${0}')
+ mathSnippet.detail = 'inline math \\( ... \\)'
+ } else {
+ mathSnippet = new vscode.CompletionItem('\\[', vscode.CompletionItemKind.Function)
+ mathSnippet.insertText = new vscode.SnippetString('${1}\\]${0}')
+ mathSnippet.detail = 'display math \\[ ... \\]'
+ }
if (vscode.workspace.getConfiguration('editor', document.uri).get('autoClosingBrackets')) {
- inlineMath.range = new vscode.Range(position, position.translate(0, 1))
+ mathSnippet.range = new vscode.Range(position, position.translate(0, 1))
}
- resolve([inlineMath])
+ resolve([mathSnippet])
return
} else {
resolve()

original_message: add latex displaymath snippet


message: support \{ snippet

diff:
@@ -12,6 +12,7 @@ export class Command {
refreshTimer: number
defaultCommands: {[key: string]: vscode.CompletionItem} = {}
newcommandData: {[id: string]: {position: vscode.Position, file: string}} = {}
+ specialBrackets: {[key: string]: vscode.CompletionItem}
constructor(extension: Extension) {
this.extension = extension
@@ -47,6 +48,13 @@ export class Command {
command.insertText = new vscode.SnippetString(item.snippet)
this.defaultCommands[key] = command
})
+ const bracketCommands = {'latexinlinemath': '(', 'latexdisplaymath': '[', 'curlybrackets': '{'}
+ this.specialBrackets = Object.keys(this.defaultCommands)
+ .filter(key => bracketCommands.hasOwnProperty(key))
+ .reduce((obj, key) => {
+ obj[bracketCommands[key]] = this.defaultCommands[key]
+ return obj
+ }, {})
}
provide() : vscode.CompletionItem[] {

diff:
@@ -49,24 +49,19 @@ export class Completer implements vscode.CompletionItemProvider {
provideCompletionItems(document: vscode.TextDocument, position: vscode.Position, _token: vscode.CancellationToken) : Promise<vscode.CompletionItem[]> {
return new Promise((resolve, _reject) => {
const invokeChar = document.lineAt(position.line).text[position.character - 1]
- if (['(', '['].indexOf(invokeChar) > -1) {
- if (position.character > 1 && document.lineAt(position.line).text[position.character - 2] === '\\') {
- let mathSnippet
- if (invokeChar === '(') {
- mathSnippet = new vscode.CompletionItem('\\(', vscode.CompletionItemKind.Function)
- mathSnippet.insertText = new vscode.SnippetString('${1}\\)${0}')
- mathSnippet.detail = 'inline math \\( ... \\)'
+ if (this.command.specialBrackets.hasOwnProperty(invokeChar)) {
+ const currentLine = document.lineAt(position.line).text
+ if (position.character > 1 && currentLine[position.character - 2] === '\\') {
+ const mathSnippet = Object.assign({}, this.command.specialBrackets[invokeChar])
+ if (vscode.workspace.getConfiguration('editor', document.uri).get('autoClosingBrackets') &&
+ (currentLine.length > position.character && [')', ']', '}'].indexOf(currentLine[position.character]) > 0)) {
+ mathSnippet.range = new vscode.Range(position.translate(0, -1), position.translate(0, 1))
} else {
- mathSnippet = new vscode.CompletionItem('\\[', vscode.CompletionItemKind.Function)
- mathSnippet.insertText = new vscode.SnippetString('${1}\\]${0}')
- mathSnippet.detail = 'display math \\[ ... \\]'
- }
- if (vscode.workspace.getConfiguration('editor', document.uri).get('autoClosingBrackets')) {
- mathSnippet.range = new vscode.Range(position, position.translate(0, 1))
+ mathSnippet.range = new vscode.Range(position.translate(0, -1), position)
}
resolve([mathSnippet])
return
- } else {
+ } else if (['(', '['].indexOf(invokeChar) > -1) {
resolve()
return
}

original_message: support \{ snippet


diff:
@@ -99,12 +99,15 @@ export class Builder {
const configuration = vscode.workspace.getConfiguration('latex-workshop')
if (!this.disableCleanAndRetry && configuration.get('latex.autoBuild.cleanAndRetry.enabled') && !configuration.get('latex.clean.enabled')) {
+ this.disableCleanAndRetry = true
+ if (signal !== 'SIGTERM') {
this.extension.logger.displayStatus('x', 'errorForeground', `Recipe terminated with error. Retry building the project.`, 'warning')
this.extension.logger.addLogMessage(`Cleaning auxillary files and retrying build after toolchain error.`)
- this.disableCleanAndRetry = true
+
this.extension.commander.clean().then(() => {
this.buildStep(rootFile, steps, 0, recipeName)
})
+ }
} else {
this.extension.logger.displayStatus('x', 'errorForeground')
const res = this.extension.logger.showErrorMessage('Recipe terminated with error.', 'Open compiler log')

original_message: do not start build for terminated process. Fix #556


message: don't lose focus for pdf tab preview

diff:
@@ -96,7 +96,13 @@ export class Viewer {
client.websocket.close()
}
this.clients[pdfFile.toLocaleUpperCase()] = {type: 'tab'}
- vscode.commands.executeCommand('vscode.previewHtml', uri, column, path.basename(pdfFile))
+ const window = vscode.window.activeTextEditor
+ vscode.commands.executeCommand('vscode.previewHtml', uri, column, path.basename(pdfFile)).then(success => {
+ if (success && window) {
+ vscode.window.showTextDocument(window.document)
+ }
+ return true
+ })
this.extension.logger.addLogMessage(`Open PDF tab for ${pdfFile}`)
}

original_message: don't lose focus for pdf tab preview


message: kill tslint error

diff:
@@ -19,10 +19,11 @@ function escapeRegExp(str) {
}
function regexpAllMatches(str: string, reg: RegExp) {
- let m
const res: any[] = []
- while (m = reg.exec(str)) {
+ let m = reg.exec(str)
+ while (m) {
res.push(m)
+ m = reg.exec(str)
}
return res
}
@@ -93,7 +94,7 @@ export class EnvPair {
if (dir === -1) {
allMatches = allMatches.reverse()
}
- for (let m of allMatches) {
+ for (const m of allMatches) {
if ((m[1] === 'begin' && dir === 1) || (m[1] === 'end' && dir === -1)) {
nested += 1
}

original_message: kill tslint error


message: support multiple root file

diff:
@@ -11,12 +11,12 @@ export class Commander {
this.extension = extension
}
- build(skipSelection: boolean = false, recipe: string | undefined = undefined) {
+ async build(skipSelection: boolean = false, recipe: string | undefined = undefined) {
this.extension.logger.addLogMessage(`BUILD command invoked.`)
if (!vscode.window.activeTextEditor || !this.extension.manager.isTex(vscode.window.activeTextEditor.document.fileName)) {
return
}
- const rootFile = this.extension.manager.findRoot()
+ const rootFile = await this.extension.manager.findRoot()
if (rootFile === undefined) {
this.extension.logger.addLogMessage(`Cannot find LaTeX root file.`)
@@ -78,12 +78,12 @@ export class Commander {
})
}
- view() {
+ async view() {
this.extension.logger.addLogMessage(`VIEW command invoked.`)
if (!vscode.window.activeTextEditor || !this.extension.manager.isTex(vscode.window.activeTextEditor.document.fileName)) {
return
}
- const rootFile = this.extension.manager.findRoot()
+ const rootFile = await this.extension.manager.findRoot()
if (rootFile === undefined) {
this.extension.logger.addLogMessage(`Cannot find LaTeX root PDF to view.`)
return
@@ -111,12 +111,12 @@ export class Commander {
this.extension.builder.kill()
}
- browser() {
+ async browser() {
this.extension.logger.addLogMessage(`BROWSER command invoked.`)
if (!vscode.window.activeTextEditor || !this.extension.manager.isTex(vscode.window.activeTextEditor.document.fileName)) {
return
}
- const rootFile = this.extension.manager.findRoot()
+ const rootFile = await this.extension.manager.findRoot()
if (rootFile !== undefined) {
this.extension.viewer.openViewer(rootFile)
} else {
@@ -124,12 +124,12 @@ export class Commander {
}
}
- tab() {
+ async tab() {
this.extension.logger.addLogMessage(`TAB command invoked.`)
if (!vscode.window.activeTextEditor || !this.extension.manager.isTex(vscode.window.activeTextEditor.document.fileName)) {
return
}
- const rootFile = this.extension.manager.findRoot()
+ const rootFile = await this.extension.manager.findRoot()
if (rootFile !== undefined) {
this.extension.viewer.openTab(rootFile)
} else {
@@ -146,18 +146,18 @@ export class Commander {
this.extension.viewer.openTab(uri.fsPath, false)
}
- synctex() {
+ async synctex() {
this.extension.logger.addLogMessage(`SYNCTEX command invoked.`)
- this.extension.manager.findRoot()
+ await this.extension.manager.findRoot()
if (!vscode.window.activeTextEditor || !this.extension.manager.isTex(vscode.window.activeTextEditor.document.fileName)) {
return
}
this.extension.locator.syncTeX()
}
- clean() : Promise<void> {
+ async clean() : Promise<void> {
this.extension.logger.addLogMessage(`CLEAN command invoked.`)
- this.extension.manager.findRoot()
+ await this.extension.manager.findRoot()
return this.extension.cleaner.clean()
}

diff:
@@ -12,9 +12,9 @@ export class Cleaner {
this.extension = extension
}
- clean() : Promise<void> {
+ async clean() : Promise<void> {
if (this.extension.manager.rootFile !== undefined) {
- this.extension.manager.findRoot()
+ await this.extension.manager.findRoot()
}
const configuration = vscode.workspace.getConfiguration('latex-workshop')
let globs = configuration.get('latex.clean.fileTypes') as string[]

diff:
@@ -12,9 +12,9 @@ export class Counter {
this.extension = extension
}
- count(file: string, merge: boolean = true) {
+ async count(file: string, merge: boolean = true) {
if (this.extension.manager.rootFile !== undefined) {
- this.extension.manager.findRoot()
+ await this.extension.manager.findRoot()
}
const configuration = vscode.workspace.getConfiguration('latex-workshop')
const args = configuration.get('texcount.args') as string[]

diff:
@@ -9,6 +9,7 @@ import {Extension} from '../main'
export class Manager {
extension: Extension
rootFiles: object
+ rootOfFiles: object
workspace: string
texFileTree: { [id: string]: Set<string> } = {}
fileWatcher: chokidar.FSWatcher
@@ -19,6 +20,7 @@ export class Manager {
this.extension = extension
this.watched = []
this.rootFiles = {}
+ this.rootOfFiles = {}
this.workspace = ''
}
@@ -27,6 +29,12 @@ export class Manager {
}
get rootFile() {
+ const window = vscode.window.activeTextEditor
+ if (window && window.document && this.rootOfFiles.hasOwnProperty(window.document.fileName)) {
+ const root = this.rootOfFiles[window.document.fileName]
+ this.rootFiles[this.workspace] = root
+ return root
+ }
return this.rootFiles[this.workspace]
}
@@ -64,16 +72,17 @@ export class Manager {
}
}
- findRoot() : string | undefined {
+ async findRoot() : Promise<string | undefined> {
this.updateWorkspace()
const findMethods = [() => this.findRootMagic(), () => this.findRootSelf(), () => this.findRootSaved(), () => this.findRootDir()]
for (const method of findMethods) {
- const rootFile = method()
+ const rootFile = await method()
if (rootFile !== undefined) {
if (this.rootFile !== rootFile) {
this.extension.logger.addLogMessage(`Root file changed from: ${this.rootFile}. Find all dependencies.`)
this.rootFile = rootFile
- this.findAllDependentFiles()
+ this.findAllDependentFiles(rootFile)
+ this.updateRootOfFiles(rootFile, rootFile)
} else {
this.extension.logger.addLogMessage(`Root file remains unchanged from: ${this.rootFile}.`)
}
@@ -83,6 +92,15 @@ export class Manager {
return undefined
}
+ updateRootOfFiles(root: string, file: string) {
+ if (this.texFileTree.hasOwnProperty(file)) {
+ this.rootOfFiles[file] = root
+ for (const f of this.texFileTree[file]) {
+ this.updateRootOfFiles(root, f)
+ }
+ }
+ }
+
findRootMagic() : string | undefined {
if (!vscode.window.activeTextEditor) {
return undefined
@@ -130,10 +148,18 @@ export class Manager {
}
findRootSaved() : string | undefined {
+ const window = vscode.window
+ if (window && window.activeTextEditor) {
+ if (this.isRoot(this.rootFile, window.activeTextEditor.document.fileName)) {
+ return this.rootFile
+ } else {
+ return undefined
+ }
+ }
return this.rootFile
}
- findRootDir() : string | undefined {
+ async findRootDir() : Promise<string | undefined> {
const regex = /\\begin{document}/m
if (!this.workspace) {
@@ -141,28 +167,50 @@ export class Manager {
}
try {
- const files = fs.readdirSync(this.workspace)
- for (let file of files) {
- if (path.extname(file) !== '.tex') {
- continue
- }
- file = path.join(this.workspace, file)
- const content = fs.readFileSync(file)
-
+ const urls = await vscode.workspace.findFiles('**/*.tex', undefined)
+ for (const url of urls) {
+ const content = fs.readFileSync(url.fsPath)
const result = content.toString().match(regex)
if (result) {
- file = path.resolve(this.workspace, file)
+ const file = url.fsPath
+ this.extension.logger.addLogMessage(`Try root file in root directory: ${file}`)
+ const window = vscode.window
+ if (window && window.activeTextEditor && this.isRoot(url.fsPath, window.activeTextEditor.document.fileName)) {
this.extension.logger.addLogMessage(`Found root file in root directory: ${file}`)
return file
}
}
- } catch (e) {}
+ }
+ } catch (e) {
+ console.log(e)
+ }
return undefined
}
- findAllDependentFiles() {
+ isRoot(root: string, file: string, start = true) : boolean {
+ if (!fs.existsSync(root)) {
+ return false
+ }
+ if (root === file) {
+ return true
+ }
+ if (start) {
+ this.findDependentFiles(root)
+ }
+ if (!this.texFileTree.hasOwnProperty(root) || !this.texFileTree.hasOwnProperty(file)) {
+ return false
+ }
+ for (const r of this.texFileTree[root]) {
+ if (this.isRoot(r, file, false)) {
+ return true
+ }
+ }
+ return false
+ }
+
+ findAllDependentFiles(rootFile: string) {
let prevWatcherClosed = false
- if (this.fileWatcher !== undefined && this.watched.indexOf(this.rootFile) < 0) {
+ if (this.fileWatcher !== undefined && this.watched.indexOf(rootFile) < 0) {
// We have an instantiated fileWatcher, but the rootFile is not being watched.
// => the user has changed the root. Clean up the old watcher so we reform it.
this.extension.logger.addLogMessage(`Root file changed -> cleaning up old file watcher.`)
@@ -172,23 +220,23 @@ export class Manager {
}
if (prevWatcherClosed || this.fileWatcher === undefined) {
- this.extension.logger.addLogMessage(`Instatiating new file watcher for ${this.rootFile}`)
- this.fileWatcher = chokidar.watch(this.rootFile)
- this.watched.push(this.rootFile)
+ this.extension.logger.addLogMessage(`Instatiating new file watcher for ${rootFile}`)
+ this.fileWatcher = chokidar.watch(rootFile)
+ this.watched.push(rootFile)
this.fileWatcher.on('change', (filePath: string) => {
this.extension.logger.addLogMessage(`File watcher: responding to change in ${filePath}`)
this.findDependentFiles(filePath)
})
- this.fileWatcher.on('unlink', (filePath: string) => {
+ this.fileWatcher.on('unlink', async (filePath: string) => {
this.extension.logger.addLogMessage(`File watcher: ${filePath} deleted.`)
this.fileWatcher.unwatch(filePath)
this.watched.splice(this.watched.indexOf(filePath), 1)
- if (filePath === this.rootFile) {
+ if (filePath === rootFile) {
this.extension.logger.addLogMessage(`Deleted ${filePath} was root - triggering root search`)
- this.findRoot()
+ await this.findRoot()
}
})
- this.findDependentFiles(this.rootFile)
+ this.findDependentFiles(rootFile)
const configuration = vscode.workspace.getConfiguration('latex-workshop')
const additionalBib = configuration.get('latex.additionalBib') as string[]
for (const bibGlob of additionalBib) {
@@ -206,7 +254,10 @@ export class Manager {
}
}
- findDependentFiles(filePath: string) {
+ findDependentFiles(filePath: string, rootDir: string | undefined = undefined) {
+ if (!rootDir) {
+ rootDir = path.dirname(filePath)
+ }
this.extension.logger.addLogMessage(`Parsing ${filePath}`)
const content = fs.readFileSync(filePath, 'utf-8')
@@ -218,7 +269,7 @@ export class Manager {
break
}
const inputFile = result[1]
- let inputFilePath = path.resolve(path.join(this.rootDir, inputFile))
+ let inputFilePath = path.resolve(path.join(rootDir, inputFile))
if (path.extname(inputFilePath) === '') {
inputFilePath += '.tex'
}
@@ -227,12 +278,12 @@ export class Manager {
}
if (fs.existsSync(inputFilePath)) {
this.texFileTree[filePath].add(inputFilePath)
- if (this.watched.indexOf(inputFilePath) < 0) {
+ if (this.fileWatcher && this.watched.indexOf(inputFilePath) < 0) {
this.extension.logger.addLogMessage(`Adding ${inputFilePath} to file watcher.`)
this.fileWatcher.add(inputFilePath)
this.watched.push(inputFilePath)
- this.findDependentFiles(inputFilePath)
}
+ this.findDependentFiles(inputFilePath, rootDir)
}
}

diff:
@@ -200,7 +200,13 @@ export async function activate(context: vscode.ExtensionContext) {
}
if (vscode.window.activeTextEditor) {
- extension.manager.findRoot()
+ extension.manager.findRoot().then(val => {
+ if (val && val !== extension.nodeProvider.root) {
+ extension.nodeProvider.root = val
+ extension.nodeProvider.refresh()
+ extension.nodeProvider.update()
+ }
+ })
}
if (e && extension.manager.isTex(e.document.fileName)) {

diff:
@@ -11,6 +11,7 @@ export class SectionNodeProvider implements vscode.TreeDataProvider<Section> {
readonly onDidChangeTreeData: vscode.Event<Section | undefined> = this._onDidChangeTreeData.event
private hierarchy: string[]
private sectionDepths: { string?: number } = {}
+ public root: string = ''
// our data source is a set multi-rooted set of trees
private ds: Section[] = []

original_message: support multiple root file


message: smart .chktexrc location

diff:
@@ -23,6 +23,7 @@ One million downloads! This project won't be successful without contributions fr
- LaTeX log parser, with errors and warnings in LaTeX build automatically reported in VS Code.
- Real-time linting of LaTeX with ChkTeX to pick up common LaTeX issues as you type.
- Code Actions (automatic fixes) are offered for many issues found by ChkTeX.
+ - Auto load `.chktexrc` configure in the following order: 1. manually configured `-l` setting in `chktex.args`; 2. `.chktexrc` file (if exists) in the same folder as the main LaTeX file; 3. `.chktexrc` file (if exists) at the project root folder.
- LaTeX file formatting.
- Auto close LaTeX environments: just call _LaTeX Workshop: Close current environment_ from the **Command Palette**. You can easily assign a shortcut to this action through the **Keyboard Shortcuts** menu, search for `latex-workshop.close-env`.
- Navigate from `\begin/\end` to the corresponding `\end/\begin`: while on the `begin` or `end` keyword, call _LaTeX Workshop: Navigate to matching begin/end_ from the **Command Palette**. To define a shortcut, search for `latex-workshop.navigate-envpair` in the **Keyboard Shortcuts** menu.

diff:
@@ -15,6 +15,28 @@ export class Linter {
this.extension = extension
}
+ get rcPath() {
+ let rcPath
+ // 0. root file folder
+ const root = this.extension.manager.documentRoot()
+ if (root) {
+ rcPath = path.join(path.dirname(root), '.chktexrc')
+ }
+ if (fs.existsSync(rcPath)) {
+ return rcPath
+ }
+
+ // 1. project root folder
+ const ws = vscode.workspace.workspaceFolders
+ if (ws && ws.length > 0) {
+ rcPath = path.join(ws[0].uri.fsPath, '.chktexrc')
+ }
+ if (fs.existsSync(rcPath)) {
+ return rcPath
+ }
+ return undefined
+ }
+
lintRootFileIfEnabled() {
const configuration = vscode.workspace.getConfiguration('latex-workshop')
const linter = configuration.get('chktex.enabled') as boolean
@@ -53,10 +75,10 @@ export class Linter {
const configuration = vscode.workspace.getConfiguration('latex-workshop')
const command = configuration.get('chktex.path') as string
- const args = configuration.get('chktex.args.active') as string[]
- if (this.extension.manager.rootDir) {
- const rcPath = path.join(this.extension.manager.rootDir, '.chktexrc')
- if (fs.existsSync(rcPath)) {
+ const args = [...(configuration.get('chktex.args.active') as string[])]
+ if (args.indexOf('-l') < 0) {
+ const rcPath = this.rcPath
+ if (rcPath) {
args.push('-l', rcPath)
}
}
@@ -83,10 +105,10 @@ export class Linter {
const configuration = vscode.workspace.getConfiguration('latex-workshop')
const command = configuration.get('chktex.path') as string
- const args = configuration.get('chktex.args.root') as string[]
- if (this.extension.manager.rootDir) {
- const rcPath = path.join(this.extension.manager.rootDir, '.chktexrc')
- if (fs.existsSync(rcPath)) {
+ const args = [...(configuration.get('chktex.args.active') as string[])]
+ if (args.indexOf('-l') < 0) {
+ const rcPath = this.rcPath
+ if (rcPath) {
args.push('-l', rcPath)
}
}

original_message: smart .chktexrc location


message: fix open pdf view tab

diff:
@@ -96,7 +96,13 @@ export class Viewer {
client.websocket.close()
}
this.clients[pdfFile.toLocaleUpperCase()] = {type: 'tab'}
- vscode.commands.executeCommand('vscode.previewHtml', uri, column, path.basename(pdfFile))
+ const editor = vscode.window.activeTextEditor
+ vscode.commands.executeCommand('vscode.previewHtml', uri, column, path.basename(pdfFile)).then(success => {
+ if (success && editor) {
+ vscode.window.showTextDocument(editor.document, editor.viewColumn)
+ }
+ return true
+ })
this.extension.logger.addLogMessage(`Open PDF tab for ${pdfFile}`)
}

original_message: fix open pdf view tab


message:
Update pyyaml for version 5.3.1 for security fix
Fixes:

old_path: Pipfile

new_path: Pipfile

license: Apache License 2.0

repo: bridgecrewio/checkov

original_message:
Update pyyaml for version 5.3.1 for security fix (#382)

Fixes: #381

Co-authored-by: robeden <robeden@granular.ag>


message:
Terraform 0.11 support
* Support for Terraform 0.11 parsing as a fallback
HCL2 parsing is attempted first with a fallback to HCL1 parsing (via `pyhcl`)
if HCL2 parsing fails.
Fixes:
* Path fix
* Merge remote-tracking branch 'upstream/master' into 409-terraform-0.11
# Conflicts:
#	Pipfile.lock

diff:
@@ -14,6 +14,7 @@ urllib3-mock = "*"
[packages]
python-hcl2 = "*"
+pyhcl = "*"
tabulate = "*"
colorama="*"
termcolor="*"

diff:
+import hcl
import hcl2
import logging
import os
@@ -20,9 +21,17 @@ class Parser:
@staticmethod
def _parse_tf_definitions(tf_file):
+ try:
with(open(tf_file, 'r')) as file:
file.seek(0)
tf_definition = hcl2.load(file)
+ except Exception as hcl2_exception:
+ try:
+ with(open(tf_file, 'r')) as file:
+ file.seek(0)
+ tf_definition = hcl.load(file)
+ except Exception:
+ raise hcl2_exception
for resource_type in tf_definition.get('resource', []):
for resource in resource_type.values():
for named_resource in resource.values():

change_type: ADD

diff:
+# Example of HCL1 syntax that won't parse in HCL2
+# (from https://github.com/amplify-education/python-hcl2/issues/13)
+
+output "ebs_sizes" {
+ value = {
+ "x" : 600,
+ "y" : 350,
+ "z" : 350,
+ "xx" : 200
+ }
+}
\ No newline at end of file

diff:
+# Example HCL2 syntax that won't parse in HCL1
+# (from https://github.com/virtuald/pyhcl/issues/74)
+
+locals {
+ map = {
+ r1 = "21.0.0.0/16"
+ r2 = "22.4.0.0/16"
+ }
+
+ sg = {
+ "test" = [
+ "r1",
+ "r2"
+ ]
+ }
+
+}
+
+resource "aws_security_group" "test" {
+ name = "test"
+ vpc_id = "vpc-xxxxxxxxx"
+
+ dynamic "ingress" {
+ for_each = local.sg.test
+ content {
+ from_port = 80
+ to_port = 80
+ protocol = "tcp"
+ cidr_blocks = split(",", lookup(local.map, ingress.value, ingress.value))
+ }
+ }
+}
\ No newline at end of file

diff:
+import json
+import os
+import unittest
+
+from checkov.terraform.parser import Parser
+
+
+class TestTerraformHcl(unittest.TestCase):
+
+ def test_hcl1_parsing(self):
+ self._load_and_test('hcl1_example.tf', 'hcl1_example_expected.json')
+
+ def test_hcl2_parsing(self):
+ self._load_and_test('hcl2_example.tf', 'hcl2_example_expected.json')
+
+ def _load_and_test(self, test_file, test_expected_file):
+ current_dir = os.path.dirname(os.path.realpath(__file__))
+
+ parser = Parser()
+ parse_errors = {}
+ result = parser.parse_file(f'{current_dir}/{test_file}', parse_errors)
+
+ with open(f'{current_dir}/{test_expected_file}', 'r') as f:
+ expected_result = json.load(f)
+
+ self.assertEqual(expected_result, result)
+
+
+if __name__ == '__main__':
+ unittest.main()

original_message:
Terraform 0.11 support (#411)

* Support for Terraform 0.11 parsing as a fallback

HCL2 parsing is attempted first with a fallback to HCL1 parsing (via `pyhcl`)
if HCL2 parsing fails.

Fixes: #409

* Path fix

* Merge remote-tracking branch 'upstream/master' into 409-terraform-0.11

# Conflicts:
#	Pipfile.lock