message:
fix docker image name
fix docker image name

old_path: docker/Dockerfile

new_path: docker/Dockerfile

diff:
@@ -57,7 +57,7 @@ COPY --from=builder $RELEASE_PATH/starcoin \
$RELEASE_PATH/starcoin_generator \
$RELEASE_PATH/starcoin_indexer \
$RELEASE_PATH/move \
- $RELEASE_PATH/startcoin_db_exporter \
+ $RELEASE_PATH/starcoin_db_exporter \
/starcoin/
CMD ["/starcoin/starcoin"]

old_path: docker/Dockerfilefromlocal

new_path: docker/Dockerfilefromlocal

diff:
@@ -13,7 +13,7 @@ COPY $RELEASE_PATH/starcoin \
$RELEASE_PATH/starcoin_faucet \
$RELEASE_PATH/starcoin_generator \
$RELEASE_PATH/starcoin_indexer \
- $RELEASE_PATH/startcoin_db_exporter \
+ $RELEASE_PATH/starcoin_db_exporter \
/starcoin/
CMD ["/starcoin/starcoin"]

original_message:
fix docker image name (#3012)

fix docker image name


message:
export and import block range
export and import block range

change_type: MODIFY

diff:
@@ -2024,7 +2024,10 @@ dependencies = [
"hex",
"serde",
"serde_json",
+ "starcoin-chain",
+ "starcoin-config",
"starcoin-crypto",
+ "starcoin-genesis",
"starcoin-storage",
"starcoin-types",
"starcoin-vm-types",

diff:
@@ -9,6 +9,9 @@ edition = "2018"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
+starcoin-config = { path = "../../config"}
+starcoin-chain = { path = "../../chain"}
+starcoin-genesis = { path = "../../genesis"}
starcoin-storage = {path = "../../storage"}
starcoin-crypto = {path = "../../commons/crypto"}
starcoin-vm-types = {path = "../../vm/types"}

diff:
@@ -28,4 +28,33 @@ OPTIONS:
-b, --block-hash <block-hash>
-n, --cf-name <cf-name> [possible values: block, block_header]
-i, --db-path <db-path> starcoin node db path. like ~/.starcoin/barnard/starcoindb/db/starcoindb
+USAGE:
+ starcoin_db_exporter export-block-range --db-path <db-path> --end <end> --net <net> --output <output> --start <start>
+
+FLAGS:
+ -h, --help Prints help information
+ -V, --version Prints version information
+
+OPTIONS:
+ -i, --db-path <db-path> starcoin node db path. like ~/.starcoin/main
+ -e, --end <end>
+ -n, --net <net> Chain Network, like main, proxima
+ -o, --output <output> output file, like block.csv
+ -s, --start <start>
+
+USAGE:
+ starcoin_db_exporter apply-block [FLAGS] --input-path <input-path> --net <net> --to-path <to-path> [verifier]
+
+FLAGS:
+ -h, --help Prints help information
+ -V, --version Prints version information
+ -w, --watch Watch metrics logs
+
+OPTIONS:
+ -o, --input-path <input-path> input file, like accounts.csv
+ -n, --net <net> Chain Network
+ -i, --to-path <to-path> starcoin node db path. like ~/.starcoin/main
+
+ARGS:
+ <verifier> Verify type: Basic, Consensus, Full, None, eg [possible values: Basic, Consensus, Full, None]
```

diff:
// Copyright (c) The Starcoin Core Contributors
// SPDX-License-Identifier: Apache-2.0
-use anyhow::{bail, Result};
+use anyhow::{bail, format_err, Result};
use bcs_ext::Sample;
use csv::Writer;
+use starcoin_chain::verifier::{
+ BasicVerifier, ConsensusVerifier, FullVerifier, NoneVerifier, Verifier,
+};
+use starcoin_chain::{BlockChain, ChainReader};
+use starcoin_config::{BuiltinNetworkID, ChainNetwork, RocksdbConfig};
use starcoin_crypto::HashValue;
+use starcoin_genesis::Genesis;
use starcoin_storage::block::FailedBlock;
+use starcoin_storage::cache_storage::CacheStorage;
use starcoin_storage::db_storage::DBStorage;
-use starcoin_storage::storage::InnerStore;
use starcoin_storage::storage::ValueCodec;
+use starcoin_storage::storage::{InnerStore, StorageInstance};
use starcoin_storage::{
- BLOCK_HEADER_PREFIX_NAME, BLOCK_PREFIX_NAME, FAILED_BLOCK_PREFIX_NAME, VEC_PREFIX_NAME,
+ Storage, BLOCK_HEADER_PREFIX_NAME, BLOCK_PREFIX_NAME, FAILED_BLOCK_PREFIX_NAME, VEC_PREFIX_NAME,
};
-use starcoin_types::block::{Block, BlockHeader};
+use starcoin_types::block::{Block, BlockHeader, BlockNumber};
use std::fmt::{Debug, Formatter};
+use std::fs::File;
+use std::io::{BufRead, BufReader, Write};
use std::path::PathBuf;
use std::str::FromStr;
+use std::sync::Arc;
+use std::time::SystemTime;
use structopt::StructOpt;
pub fn export<W: std::io::Write>(
@@ -162,6 +173,8 @@ struct Opt {
enum Cmd {
Exporter(ExporterOptions),
Checkkey(CheckKeyOptions),
+ ExportBlockRange(ExportBlockRangeOptions),
+ ApplyBlock(ApplyBlockOptions),
}
#[derive(Debug, Clone, StructOpt)]
@@ -192,6 +205,44 @@ pub struct CheckKeyOptions {
pub block_hash: HashValue,
}
+#[derive(Debug, Clone, StructOpt)]
+#[structopt(name = "export-block-range", about = "export block range")]
+pub struct ExportBlockRangeOptions {
+ #[structopt(long, short = "n")]
+ /// Chain Network, like main, proxima
+ pub net: BuiltinNetworkID,
+ #[structopt(long, short = "o", parse(from_os_str))]
+ /// output file, like block.csv
+ pub output: PathBuf,
+ #[structopt(long, short = "i", parse(from_os_str))]
+ /// starcoin node db path. like ~/.starcoin/main
+ pub db_path: PathBuf,
+ #[structopt(long, short = "s")]
+ pub start: BlockNumber,
+ #[structopt(long, short = "e")]
+ pub end: BlockNumber,
+}
+
+#[derive(Debug, StructOpt)]
+#[structopt(name = "apply-block-range", about = "apply block range")]
+pub struct ApplyBlockOptions {
+ #[structopt(long, short = "n")]
+ /// Chain Network
+ pub net: BuiltinNetworkID,
+ #[structopt(long, short = "i", parse(from_os_str))]
+ /// starcoin node db path. like ~/.starcoin/main
+ pub to_path: PathBuf,
+ #[structopt(long, short = "o", parse(from_os_str))]
+ /// input file, like accounts.csv
+ pub input_path: PathBuf,
+ #[structopt(possible_values = &Verifier::variants(), case_insensitive = true)]
+ /// Verify type: Basic, Consensus, Full, None, eg.
+ pub verifier: Option<Verifier>,
+ #[structopt(long, short = "w")]
+ /// Watch metrics logs.
+ pub watch: bool,
+}
+
fn main() -> anyhow::Result<()> {
let opt = Opt::from_args();
let cmd = match opt.cmd {
@@ -266,5 +317,96 @@ fn main() -> anyhow::Result<()> {
}
return Ok(());
}
+
+ if let Cmd::ExportBlockRange(option) = cmd {
+ let result = export_block_range(
+ option.db_path,
+ option.output,
+ option.net,
+ option.start,
+ option.end,
+ );
+ return result;
+ }
+
+ if let Cmd::ApplyBlock(option) = cmd {
+ let verifier = option.verifier.unwrap_or(Verifier::Basic);
+ let result = apply_blocks(option.to_path, option.input_path, option.net, verifier);
+ return result;
+ }
+ Ok(())
+}
+
+pub fn export_block_range(
+ from_dir: PathBuf,
+ output: PathBuf,
+ network: BuiltinNetworkID,
+ start: BlockNumber,
+ end: BlockNumber,
+) -> anyhow::Result<()> {
+ let net = ChainNetwork::new_builtin(network);
+ let db_stoarge = DBStorage::new(
+ from_dir.join("starcoindb/db"),
+ RocksdbConfig::default(),
+ None,
+ )?;
+ let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
+ CacheStorage::new(None),
+ db_stoarge,
+ ))?);
+ let (chain_info, _) =
+ Genesis::init_and_check_storage(&net, storage.clone(), from_dir.as_ref())?;
+ let chain = BlockChain::new(net.time_service(), chain_info.head().id(), storage, None)
+ .expect("create block chain should success.");
+ let start_time = SystemTime::now();
+ let block_list: Result<Vec<Block>> = (start..end)
+ .collect::<Vec<BlockNumber>>()
+ .into_iter()
+ .map(|num| {
+ chain
+ .get_block_by_number(num)?
+ .ok_or_else(|| format_err!("{} get block error", num))
+ })
+ .collect();
+ let block_list = block_list?;
+ let mut file = File::create(output)?;
+ for block in block_list {
+ writeln!(file, "{}", serde_json::to_string(&block)?)?;
+ }
+ file.flush()?;
+ let use_time = SystemTime::now().duration_since(start_time)?;
+ println!("export range block use time: {:?}", use_time.as_nanos());
+ Ok(())
+}
+
+pub fn apply_blocks(
+ to_dir: PathBuf,
+ input_path: PathBuf,
+ network: BuiltinNetworkID,
+ verifier: Verifier,
+) -> anyhow::Result<()> {
+ let net = ChainNetwork::new_builtin(network);
+ let db_stoarge = DBStorage::new(to_dir.join("starcoindb/db"), RocksdbConfig::default(), None)?;
+ let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
+ CacheStorage::new(None),
+ db_stoarge,
+ ))?);
+ let (chain_info, _) = Genesis::init_and_check_storage(&net, storage.clone(), to_dir.as_ref())?;
+ let mut chain = BlockChain::new(net.time_service(), chain_info.head().id(), storage, None)
+ .expect("create block chain should success.");
+ let start_time = SystemTime::now();
+ let reader = BufReader::new(File::open(input_path)?);
+ for record in reader.lines() {
+ let record = record?;
+ let block: Block = serde_json::from_str(record.as_str())?;
+ match verifier {
+ Verifier::Basic => chain.apply_with_verifier::<BasicVerifier>(block)?,
+ Verifier::Consensus => chain.apply_with_verifier::<ConsensusVerifier>(block)?,
+ Verifier::Full => chain.apply_with_verifier::<FullVerifier>(block)?,
+ Verifier::None => chain.apply_with_verifier::<NoneVerifier>(block)?,
+ };
+ }
+ let use_time = SystemTime::now().duration_since(start_time)?;
+ println!("apply blocks use time: {:?}", use_time.as_nanos());
Ok(())
}

license: Apache License 2.0

repo: starcoinorg/starcoin

original_message:
export and import block range (#3017)

export and import block range


message:
Create import_main_net_block.sh
* Create import_main_net_block.sh
import main net block shell
* Update import_main_net_block.sh
* Update release.sh
* Update release.sh

change_type: ADD

diff:
+#!/bin/bash
+
+STEP=500000
+main_cnt=$(ps -ef | grep starcoin | grep main | grep -v export | grep -v grep | wc -l)
+echo $main_cnt
+if [ $main_cnt -ne 0 ]; then
+ echo "main_net running, exit"
+ exit 1
+fi
+for((i = 0; i < 5; i++));
+do
+ start=$((i * STEP + 1))
+ end=$((i * STEP + STEP))
+ name=block_"$start"_"$end".csv
+ url=https://s3.ap-northeast-1.amazonaws.com/main.starcoin.org/$name
+ wget $url
+ ./starcoin_db_exporter apply-block -i $name -n main -o ~/.starcoin/main
+done
+wget https://s3.ap-northeast-1.amazonaws.com/main.starcoin.org/block_2500001_2890710.csv
+./starcoin_db_exporter apply-block -i block_2500001_2890710.csv -n main -o ~/.starcoin/main/
+

diff:
@@ -6,6 +6,7 @@ cp -v target/release/starcoin_miner starcoin-artifacts/
cp -v target/release/starcoin_generator starcoin-artifacts/
cp -v target/release/move starcoin-artifacts/
cp -v target/release/starcoin_db_exporter starcoin-artifacts/
+cp -v scripts/import_main_net_block.sh starcoin_artifacts/
if [ "$1" == "windows-latest" ]; then
7z a -r starcoin-$1.zip starcoin-artifacts
else

original_message:
Create import_main_net_block.sh (#3042)

* Create import_main_net_block.sh

import main net block shell

* Update import_main_net_block.sh

* Update release.sh

* Update release.sh


message:
benchmark flamegraph save on s3
benchmark flamegraph svg save on s3
1. pull request use pull request number as s3 folder name
2. benchmark branch use commit short sha as  s3 folder name

diff:
@@ -19,11 +19,15 @@ jobs:
components: rustfmt,clippy
- name: setup environment
run: bash ./scripts/dev_setup.sh -b -t -y
- - name: run cargo bench
- run: bash ./scripts/benchmark.sh
+ - name: add commit short sha
+ run: echo "SHORT_SHA=`git rev-parse --short=7 HEAD`" >> $GITHUB_ENV
+ - name: run flamegraph
+ run: bash ./scripts/flamegraph.sh ${SHORT_SHA}
env:
RUST_LOG: error
RUST_BACKTRACE: full
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- name: upload Artifact
uses: actions/upload-artifact@v2
with:

diff:
@@ -12,7 +12,15 @@ jobs:
submodules: recursive
- name: setup rust toolchain
uses: actions-rs/toolchain@v1
- - uses: boa-dev/criterion-compare-action@master
+ - name: run flamegraph
+ run: bash ./scripts/flamegraph.sh ${{ github.event.pull_request.number }}
+ env:
+ RUST_LOG: error
+ RUST_BACKTRACE: full
+ AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ - name: use criterion
+ uses: boa-dev/criterion-compare-action@master
env:
RUST_LOG: error
RUST_BACKTRACE: full
@@ -21,3 +29,4 @@ jobs:
branchName: ${{ github.base_ref }}
# benchName: "bench_state_tree" # Optional. Compare only this benchmark target
token: ${{ secrets.GITHUB_TOKEN }}
+

diff:
+#!/bin/bash
+
+STARCOIN_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && cd .. && pwd)"
+
+PR_NUMBER=$1
+
+if [[ "$(uname)" != "Linux" ]]; then
+ echo "run flamegraph only in linux. exit"
+fi
+
+echo "PR_NUMBER $PR_NUMBER"
+
+cmd="RUST_LOG=error cargo bench --bench bench_storage -- --profile-time=10"
+echo "run flamegraph with cmd: ${cmd}"
+eval "$cmd"
+
+cmd="RUST_LOG=error cargo bench --bench bench_chain -- --profile-time=10"
+echo "run flamegraph with cmd: ${cmd}"
+eval "$cmd"
+
+cmd="RUST_LOG=error cargo bench --bench bench_state_tree -- --profile-time=10"
+echo "run flamegraph with cmd: ${cmd}"
+eval "$cmd"
+
+cmd="RUST_LOG=error cargo bench --bench bench_vm -- --profile-time=10"
+echo "run flamegraph with cmd: ${cmd}"
+eval "$cmd"
+
+cmd="RUST_LOG=error cargo build --release --bin=starcoin_db_exporter"
+echo "run build with cmd: ${cmd}"
+eval "$cmd"
+
+
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"accumulator_append.svg" --body $STARCOIN_DIR/target/criterion/accumulator_append/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"storage_transaction.svg" --body $STARCOIN_DIR/target/criterion/storage_transaction/profile/flamegraph.svg
+
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"query_block_in10_times100.svg" --body $STARCOIN_DIR/target/criterion/query_block/"query_block_in(10)_times(100)"/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"query_block_in10_times1000.svg" --body $STARCOIN_DIR/target/criterion/query_block/"query_block_in(10)_times(1000)"/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"query_block_in10_times10000.svg" --body $STARCOIN_DIR/target/criterion/query_block/"query_block_in(10)_times(10000)"/profile/flamegraph.svg
+
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"query_block_in1000_times100.svg" --body $STARCOIN_DIR/target/criterion/query_block/"query_block_in(1000)_times(100)"/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"query_block_in1000_times1000.svg" --body $STARCOIN_DIR/target/criterion/query_block/"query_block_in(1000)_times(1000)"/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"query_block_in1000_times10000.svg" --body $STARCOIN_DIR/target/criterion/query_block/"query_block_in(1000)_times(10000)"/profile/flamegraph.svg
+
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"get_with_proof_mem_store.svg" --body $STARCOIN_DIR/target/criterion/get_with_proof/mem_store/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"get_with_proof_db_store.svg" --body $STARCOIN_DIR/target/criterion/get_with_proof/db_store/profile/flamegraph.svg
+
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"put_and_commit_db_store_1.svg" --body $STARCOIN_DIR/target/criterion/put_and_commit/db_store/1/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"put_and_commit_db_store_10.svg" --body $STARCOIN_DIR/target/criterion/put_and_commit/db_store/10/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"put_and_commit_db_store_100.svg" --body $STARCOIN_DIR/target/criterion/put_and_commit/db_store/100/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"put_and_commit_db_store_5.svg" --body $STARCOIN_DIR/target/criterion/put_and_commit/db_store/5/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"put_and_commit_db_store_50.svg" --body $STARCOIN_DIR/target/criterion/put_and_commit/db_store/50/profile/flamegraph.svg
+
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"put_and_commit_mem_store_1.svg" --body $STARCOIN_DIR/target/criterion/put_and_commit/mem_store/1/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"put_and_commit_mem_store_10.svg" --body $STARCOIN_DIR/target/criterion/put_and_commit/mem_store/10/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"put_and_commit_mem_store_100.svg" --body $STARCOIN_DIR/target/criterion/put_and_commit/mem_store/100/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"put_and_commit_mem_store_5.svg" --body $STARCOIN_DIR/target/criterion/put_and_commit/mem_store/5/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"put_and_commit_mem_store_50.svg" --body $STARCOIN_DIR/target/criterion/put_and_commit/mem_store/50/profile/flamegraph.svg
+
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"transaction_execution_1.svg" --body $STARCOIN_DIR/target/criterion/vm/transaction_execution/1/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"transaction_execution_10.svg" --body $STARCOIN_DIR/target/criterion/vm/transaction_execution/10/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"transaction_execution_20.svg" --body $STARCOIN_DIR/target/criterion/vm/transaction_execution/20/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"transaction_execution_5.svg" --body $STARCOIN_DIR/target/criterion/vm/transaction_execution/5/profile/flamegraph.svg
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"transaction_execution_50.svg" --body $STARCOIN_DIR/target/criterion/vm/transaction_execution/50/profile/flamegraph.svg
+
+cd /tmp
+rm -f block_1_10000.csv
+wget https://s3.ap-northeast-1.amazonaws.com/main.starcoin.org/block_1_10000.csv
+mkdir -p /tmp/$PR_NUMBER/main
+rm -rf /tmp/$PR_NUMBER/main/*
+$STARCOIN_DIR/target/release/starcoin_db_exporter apply-block -i /tmp/block_1_10000.csv -n main -o /tmp/$PR_NUMBER/main
+
+aws s3api put-object --bucket flamegraph.starcoin.org --key "$PR_NUMBER"/"apply_block.svg" --body /tmp/flamegraph.svg
+
+rm -rf /tmp/$PR_NUMBER/
\ No newline at end of file

original_message:
benchmark flamegraph save on s3 (#3043)

benchmark flamegraph svg save on s3
1. pull request use pull request number as s3 folder name
2. benchmark branch use commit short sha as  s3 folder name


diff:
@@ -4902,9 +4902,9 @@ dependencies = [
[[package]]
name = "lru"
-version = "0.7.1"
+version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "469898e909a1774d844793b347135a0cd344ca2f69d082013ecb8061a2229a3a"
+checksum = "6c748cfe47cb8da225c37595b3108bea1c198c84aaae8ea0ba76d01dda9fc803"
dependencies = [
"hashbrown 0.11.2",
]
@@ -5722,7 +5722,7 @@ dependencies = [
"linked-hash-map",
"linked_hash_set",
"log 0.4.14",
- "lru 0.7.1",
+ "lru 0.7.0",
"network-p2p-types",
"once_cell",
"parking_lot 0.11.2",
@@ -8596,7 +8596,7 @@ dependencies = [
"bcs-ext",
"byteorder 1.4.3",
"itertools 0.10.3",
- "lru 0.7.1",
+ "lru 0.7.0",
"mirai-annotations",
"once_cell",
"parking_lot 0.11.2",
@@ -9385,7 +9385,7 @@ dependencies = [
"futures-timer",
"hex",
"log 0.4.14",
- "lru 0.7.1",
+ "lru 0.7.0",
"network-api",
"network-p2p",
"network-p2p-types",
@@ -9881,7 +9881,7 @@ dependencies = [
"anyhow",
"bcs-ext",
"forkable-jellyfish-merkle",
- "lru 0.7.1",
+ "lru 0.7.0",
"parking_lot 0.11.2",
"serde",
"starcoin-crypto",
@@ -9903,7 +9903,7 @@ dependencies = [
"chrono",
"coarsetime",
"forkable-jellyfish-merkle",
- "lru 0.7.1",
+ "lru 0.7.0",
"num_enum",
"once_cell",
"parking_lot 0.11.2",

diff:
@@ -17,7 +17,7 @@ logger = {path = "../../commons/logger", package="starcoin-logger"}
starcoin-crypto = { package="starcoin-crypto", path = "../../commons/crypto"}
bcs-ext = { package="bcs-ext", path = "../../commons/bcs_ext" }
serde = { version = "1.0.130" }
-lru = "0.7.1"
+lru = "0.7.0"
parking_lot = "0.11.2"
schemars = {git = "https://github.com/starcoinorg/schemars", rev="3f8ecc27f02b27d56c716de58d7880ae7816390c"}

diff:
@@ -33,7 +33,7 @@ bitflags = "1.3.2"
tempfile = "3.1.0"
rand = "0.8.4"
parking_lot = "0.11.2"
-lru = "0.7.1"
+lru = "0.7.0"
serde = { version = "1.0.130", features = ["derive"] }
serde_json = { version="1.0", features = ["arbitrary_precision"]}

diff:
@@ -10,7 +10,7 @@ edition = "2018"
anyhow = "1.0.41"
thiserror = "1.0"
parking_lot = "0.11.2"
-lru = "0.7.1"
+lru = "0.7.0"
starcoin-types = {path = "../../types"}
starcoin-vm-types = {path = "../../vm/types"}
starcoin-state-api = {path = "../api"}

diff:
@@ -15,7 +15,7 @@ crypto = { package="starcoin-crypto", path = "../commons/crypto"}
bcs-ext = { package="bcs-ext", path = "../commons/bcs_ext" }
chrono = "0.4"
byteorder = "1.4.3"
-lru = "0.7.1"
+lru = "0.7.0"
parking_lot = "0.11.2"
proptest = { version = "1.0.0", optional = true }
proptest-derive = { version = "0.3.0", optional = true }

original_message: revert lru version to fix use-after-free (#3107)


message: fix compile windows error

diff:
-// Copyright (c) 2018-2021, StarCoin Project
-//
-// All rights reserved.
-//
-// Redistribution and use in source and binary forms, with or without modification, are
-// permitted provided that the following conditions are met:
-//
-// 1. Redistributions of source code must retain the above copyright notice, this list of
-// conditions and the following disclaimer.
-//
-// 2. Redistributions in binary form must reproduce the above copyright notice, this list
-// of conditions and the following disclaimer in the documentation and/or other
-// materials provided with the distribution.
-//
-// 3. Neither the name of the copyright holder nor the names of its contributors may be
-// used to endorse or promote products derived from this software without specific
-// prior written permission.
-//
-// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY
-// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
-// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
-// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
-// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
-// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
-// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-//
-// Parts of this file are originally copyright (c) 2012-2013 The Cryptonote developers
-
#include <assert.h>
#include <stddef.h>
#include <stdint.h>
@@ -324,7 +294,7 @@ static inline int force_software_aes(void)
} while (0)
-#if !defined NO_AES && (defined(__x86_64__) || (defined(_MSC_VER) && defined(_WIN64)))
+#if !defined NO_AES && (defined(__x86_64__) && (!defined(_MSC_VER)))
// Optimised code below, uses x86-specific intrinsics, SSE2, AES-NI
@@ -765,6 +735,7 @@ void cn_slow_hash_free_state(void)
*/
void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height)
{
+ // printf("this intel function\n");
RDATA_ALIGN16 uint8_t expandedKey[240]; /* These buffers are aligned to use later with SSE functions */
uint8_t text[INIT_SIZE_BYTE];
@@ -1198,6 +1169,7 @@ STATIC INLINE void xor_blocks(uint8_t* a, const uint8_t* b)
void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height)
{
+ //printf("this arm function\n");
RDATA_ALIGN16 uint8_t expandedKey[240];
uint8_t *local_hp_state;
@@ -1662,6 +1634,7 @@ union cn_slow_hash_state {
#define FORCE_USE_HEAP
void cn_slow_hash(const void *data, size_t length, char *hash, int variant, int prehashed, uint64_t height) {
+ //printf("this windows function\n");
#ifndef FORCE_USE_HEAP
uint8_t long_state[MEMORY];
#else

original_message: fix compile windows error (#3119)


message: fix starcoin-main.yaml format

diff:
@@ -23,7 +23,7 @@ spec:
starcoin/node-pool: seed-pool
containers:
- name: starcoin
- image: starcoin/starcoin:v1.9.0-rc.2
+ image: starcoin/starcoin:v1.9.1
imagePullPolicy: Always
command:
- bash

diff:
@@ -23,7 +23,7 @@ spec:
starcoin/node-pool: seed-pool
containers:
- name: starcoin
- image: starcoin/starcoin:v1.9.0
+ image: starcoin/starcoin:v1.9.1
imagePullPolicy: Always
command:
- bash

original_message: fix starcoin-main.yaml format (#3125)


message:
startupinfo block number back off to fix disk full wal recovery
startupinfo head back off to fix disk full corruption(issue 3128)

diff:
@@ -33,6 +33,7 @@ use std::time::SystemTime;
use structopt::StructOpt;
const BLOCK_GAP: u64 = 1000;
+const BACK_SIZE: u64 = 10000;
pub fn export<W: std::io::Write>(
db: &str,
@@ -187,6 +188,7 @@ enum Cmd {
Checkkey(CheckKeyOptions),
ExportBlockRange(ExportBlockRangeOptions),
ApplyBlock(ApplyBlockOptions),
+ StartupInfoBack(StartupInfoBackOptions),
}
#[derive(Debug, Clone, StructOpt)]
@@ -255,6 +257,20 @@ pub struct ApplyBlockOptions {
pub watch: bool,
}
+#[derive(Debug, StructOpt)]
+#[structopt(name = "startup_info_back", about = "startup info back")]
+pub struct StartupInfoBackOptions {
+ #[structopt(long, short = "n")]
+ /// Chain Network
+ pub net: BuiltinNetworkID,
+ #[structopt(long, short = "o", parse(from_os_str))]
+ /// starcoin node db path. like ~/.starcoin/main
+ pub to_path: PathBuf,
+ /// startupinfo BlockNumber back off size
+ #[structopt(long, short = "b")]
+ pub back_size: Option<u64>,
+}
+
fn main() -> anyhow::Result<()> {
let opt = Opt::from_args();
let cmd = match opt.cmd {
@@ -355,6 +371,10 @@ fn main() -> anyhow::Result<()> {
}
return result;
}
+ if let Cmd::StartupInfoBack(option) = cmd {
+ let result = startup_info_back(option.to_path, option.back_size, option.net);
+ return result;
+ }
Ok(())
}
@@ -518,3 +538,52 @@ pub fn apply_block(
println!("apply block use time: {:?}", use_time.as_secs());
Ok(())
}
+
+pub fn startup_info_back(
+ to_dir: PathBuf,
+ back_size: Option<u64>,
+ network: BuiltinNetworkID,
+) -> anyhow::Result<()> {
+ let net = ChainNetwork::new_builtin(network);
+ let db_stoarge = DBStorage::new(to_dir.join("starcoindb/db"), RocksdbConfig::default(), None)?;
+ let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
+ CacheStorage::new(None),
+ db_stoarge,
+ ))?);
+ let (chain_info, _) = Genesis::init_and_check_storage(&net, storage.clone(), to_dir.as_ref())?;
+ let chain = BlockChain::new(
+ net.time_service(),
+ chain_info.head().id(),
+ storage.clone(),
+ None,
+ )
+ .expect("create block chain should success.");
+
+ let cur_num = chain.status().head().number();
+ let back_size = back_size.unwrap_or(BACK_SIZE);
+ let back_size = if back_size < BACK_SIZE {
+ BACK_SIZE
+ } else {
+ back_size
+ };
+
+ if cur_num <= back_size {
+ println!(
+ "startup_info block number {} <= back_size {}",
+ cur_num, back_size
+ );
+ return Ok(());
+ }
+ let block_number = cur_num - back_size;
+ let block = chain
+ .get_block_by_number(block_number)?
+ .ok_or_else(|| format_err!("{} get block error", block_number))?;
+ let block_hash = block.header().id();
+ let startup_info = StartupInfo::new(block_hash);
+ storage.save_startup_info(startup_info)?;
+ println!(
+ "startup_info block number origin {} now {}",
+ cur_num, block_number
+ );
+ Ok(())
+}

original_message:
startupinfo block number back off to fix disk full wal recovery (#3129)

startupinfo head back off to fix disk full corruption(issue 3128) #3129


diff:
@@ -941,17 +941,6 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70"
-[[package]]
-name = "bzip2-sys"
-version = "0.1.11+1.0.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
-]
-
[[package]]
name = "cache-padded"
version = "1.2.0"
@@ -4763,16 +4752,14 @@ dependencies = [
[[package]]
name = "librocksdb-sys"
-version = "0.6.1+6.28.2"
+version = "6.20.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81bc587013734dadb7cf23468e531aa120788b87243648be42e2d3a072186291"
+checksum = "c309a9d2470844aceb9a4a098cf5286154d20596868b75a6b36357d2bb9ca25d"
dependencies = [
"bindgen",
- "bzip2-sys",
"cc",
"glob",
"libc",
- "libz-sys",
]
[[package]]
@@ -8190,9 +8177,9 @@ dependencies = [
[[package]]
name = "rocksdb"
-version = "0.18.0"
+version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "620f4129485ff1a7128d184bc687470c21c7951b64779ebc9cfdad3dcd920290"
+checksum = "7a62eca5cacf2c8261128631bed9f045598d40bfbe4b29f5163f0f802f8f44a7"
dependencies = [
"libc",
"librocksdb-sys",

diff:
@@ -33,7 +33,7 @@ starcoin-uint = { path = "../types/uint"}
coarsetime = "0.1.21"
[dependencies.rocksdb]
-version = "0.18"
+version = "0.17"
default-features = false
features = ["lz4"]

original_message: Mac M1 compile error revert rocksdb version to 0.17 (#3241)


message: fix starcoin_db_export apply file limit

diff:
@@ -16,7 +16,7 @@ use starcoin_account_service::{AccountEventService, AccountService, AccountStora
use starcoin_block_relayer::BlockRelayer;
use starcoin_chain_notify::ChainNotifyHandlerService;
use starcoin_chain_service::ChainReaderService;
-use starcoin_config::{check_open_fds_limit, NodeConfig};
+use starcoin_config::NodeConfig;
use starcoin_executor::VMMetrics;
use starcoin_genesis::{Genesis, GenesisError};
use starcoin_logger::prelude::*;
@@ -57,8 +57,6 @@ use starcoin_types::system_events::SystemStarted;
use std::sync::Arc;
use std::time::{Duration, SystemTime};
-const RES_FDS: u64 = 4096;
-
pub struct NodeService {
registry: ServiceRef<RegistryService>,
}
@@ -279,7 +277,6 @@ impl NodeService {
"rocksdb max open files {}",
config.storage.rocksdb_config().max_open_files
);
- check_open_fds_limit(config.storage.rocksdb_config().max_open_files as u64 + RES_FDS)?;
let mut storage_instance = StorageInstance::new_cache_and_db_instance(
CacheStorage::new_with_capacity(config.storage.cache_size(), storage_metrics.clone()),
DBStorage::new(

diff:
@@ -8,11 +8,13 @@ use crate::storage::{ColumnFamilyName, InnerStore, KeyCodec, ValueCodec, WriteOp
use crate::{StorageVersion, DEFAULT_PREFIX_NAME};
use anyhow::{ensure, format_err, Error, Result};
use rocksdb::{Options, ReadOptions, WriteBatch as DBWriteBatch, WriteOptions, DB};
-use starcoin_config::RocksdbConfig;
+use starcoin_config::{check_open_fds_limit, RocksdbConfig};
use std::collections::HashSet;
use std::marker::PhantomData;
use std::path::Path;
+const RES_FDS: u64 = 4096;
+
#[allow(clippy::upper_case_acronyms)]
pub struct DBStorage {
db: DB,
@@ -92,7 +94,7 @@ impl DBStorage {
rocksdb_opts.create_missing_column_families(true);
Self::open_inner(&rocksdb_opts, path, column_families.clone())?
};
-
+ check_open_fds_limit(rocksdb_config.max_open_files as u64 + RES_FDS)?;
Ok(DBStorage {
db,
cfs: column_families,

original_message: fix starcoin_db_export apply file limit (#3261)


diff:
@@ -171,15 +171,6 @@ impl BlockVerifier for BasicVerifier {
new_block_header.number()
);
- verify_block!(
- VerifyBlockField::Header,
- current_id == new_block_parent,
- "Invalid block: Parent id mismatch, expect:{}, got: {}, number:{}.",
- current_id,
- new_block_parent,
- new_block_header.number()
- );
-
verify_block!(
VerifyBlockField::Header,
new_block_header.timestamp() > current.timestamp(),

diff:
@@ -202,13 +202,11 @@ fn product_a_block(branch: &BlockChain, miner: &AccountInfo, uncles: Vec<BlockHe
}
#[stest::test(timeout = 120)]
-#[allow(clippy::vec_init_then_push)]
fn test_uncle() {
let (mut mock_chain, _, uncle_block_header) = gen_uncle();
let miner = mock_chain.miner();
// 3. mock chain apply
- let mut uncles = Vec::new();
- uncles.push(uncle_block_header.clone());
+ let uncles = vec![uncle_block_header.clone()];
let block = product_a_block(mock_chain.head(), miner, uncles);
mock_chain.apply(block).unwrap();
assert!(mock_chain.head().head_block().block.uncles().is_some());
@@ -223,13 +221,11 @@ fn test_uncle() {
}
#[stest::test(timeout = 120)]
-#[allow(clippy::vec_init_then_push)]
fn test_uncle_exist() {
let (mut mock_chain, _, uncle_block_header) = gen_uncle();
let miner = mock_chain.miner().clone();
// 3. mock chain apply
- let mut uncles = Vec::new();
- uncles.push(uncle_block_header.clone());
+ let uncles = vec![uncle_block_header.clone()];
let block = product_a_block(mock_chain.head(), &miner, uncles);
mock_chain.apply(block).unwrap();
assert!(mock_chain.head().head_block().block.uncles().is_some());
@@ -243,14 +239,12 @@ fn test_uncle_exist() {
assert_eq!(mock_chain.head().current_epoch_uncles_size(), 1);
// 4. uncle exist
- let mut uncles = Vec::new();
- uncles.push(uncle_block_header);
+ let uncles = vec![uncle_block_header];
let block = product_a_block(mock_chain.head(), &miner, uncles);
assert!(mock_chain.apply(block).is_err());
}
#[stest::test(timeout = 120)]
-#[allow(clippy::vec_init_then_push)]
fn test_uncle_son() {
let (mut mock_chain, mut fork_block_chain, _) = gen_uncle();
let miner = mock_chain.miner();
@@ -260,36 +254,31 @@ fn test_uncle_son() {
fork_block_chain.apply(uncle_son).unwrap();
// 4. mock chain apply
- let mut uncles = Vec::new();
- uncles.push(uncle_son_block_header);
+ let uncles = vec![uncle_son_block_header];
let block = product_a_block(mock_chain.head(), miner, uncles);
assert!(mock_chain.apply(block).is_err());
assert_eq!(mock_chain.head().current_epoch_uncles_size(), 0);
}
#[stest::test(timeout = 120)]
-#[allow(clippy::vec_init_then_push)]
fn test_random_uncle() {
let (mut mock_chain, _, _) = gen_uncle();
let miner = mock_chain.miner();
// 3. random BlockHeader and apply
- let mut uncles = Vec::new();
- uncles.push(BlockHeader::random());
+ let uncles = vec![BlockHeader::random()];
let block = product_a_block(mock_chain.head(), miner, uncles);
assert!(mock_chain.apply(block).is_err());
assert_eq!(mock_chain.head().current_epoch_uncles_size(), 0);
}
#[stest::test(timeout = 480)]
-#[allow(clippy::vec_init_then_push)]
fn test_switch_epoch() {
let (mut mock_chain, _, uncle_block_header) = gen_uncle();
let miner = mock_chain.miner().clone();
// 3. mock chain apply
- let mut uncles = Vec::new();
- uncles.push(uncle_block_header.clone());
+ let uncles = vec![uncle_block_header.clone()];
let block = product_a_block(mock_chain.head(), &miner, uncles);
mock_chain.apply(block).unwrap();
assert!(mock_chain.head().head_block().block.uncles().is_some());
@@ -322,7 +311,6 @@ fn test_switch_epoch() {
}
#[stest::test(timeout = 480)]
-#[allow(clippy::vec_init_then_push)]
fn test_uncle_in_diff_epoch() {
let (mut mock_chain, _, uncle_block_header) = gen_uncle();
let miner = mock_chain.miner().clone();
@@ -347,8 +335,7 @@ fn test_uncle_in_diff_epoch() {
assert_eq!(mock_chain.head().current_epoch_uncles_size(), 0);
// 5. mock chain apply
- let mut uncles = Vec::new();
- uncles.push(uncle_block_header);
+ let uncles = vec![uncle_block_header];
let block = product_a_block(mock_chain.head(), &miner, uncles);
assert!(mock_chain.apply(block).is_err());
}

diff:
@@ -722,7 +722,6 @@ impl NaiveInternalNode {
BinaryTreeNode::new_internal(begin, width, left, right)
}
- #[allow(clippy::vec_init_then_push)]
fn get_child_with_siblings(&self, n: u8) -> (Option<NodeKey>, Vec<HashValue>) {
let mut current_node = Rc::clone(&self.root);
let mut siblings = Vec::new();

diff:
@@ -18,7 +18,6 @@ pub use starcoin_types::account::*;
/// Returns a transaction to transfer coin from one account to another (possibly new) one, with the
/// given arguments.
-#[allow(clippy::vec_init_then_push)]
pub fn peer_to_peer_txn(
sender: &Account,
receiver: &Account,
@@ -27,9 +26,10 @@ pub fn peer_to_peer_txn(
expiration_timestamp_secs: u64,
chain_id: ChainId,
) -> SignedUserTransaction {
- let mut args: Vec<Vec<u8>> = Vec::new();
- args.push(bcs_ext::to_bytes(receiver.address()).unwrap());
- args.push(bcs_ext::to_bytes(&transfer_amount).unwrap());
+ let args = vec![
+ bcs_ext::to_bytes(receiver.address()).unwrap(),
+ bcs_ext::to_bytes(&transfer_amount).unwrap(),
+ ];
// get a SignedTransaction
sender.create_signed_txn_with_args(
@@ -51,7 +51,6 @@ pub fn peer_to_peer_txn(
}
/// Returns a transaction to create a new account with the given arguments.
-#[allow(clippy::vec_init_then_push)]
pub fn create_account_txn_sent_as_association(
new_account: &Account,
seq_num: u64,
@@ -59,10 +58,11 @@ pub fn create_account_txn_sent_as_association(
expiration_timstamp_secs: u64,
net: &ChainNetwork,
) -> SignedUserTransaction {
- let mut args: Vec<Vec<u8>> = Vec::new();
- args.push(bcs_ext::to_bytes(new_account.address()).unwrap());
- args.push(bcs_ext::to_bytes(&new_account.auth_key().to_vec()).unwrap());
- args.push(bcs_ext::to_bytes(&initial_amount).unwrap());
+ let args = vec![
+ bcs_ext::to_bytes(new_account.address()).unwrap(),
+ bcs_ext::to_bytes(&new_account.auth_key().to_vec()).unwrap(),
+ bcs_ext::to_bytes(&initial_amount).unwrap(),
+ ];
create_signed_txn_with_association_account(
TransactionPayload::ScriptFunction(ScriptFunction::new(

diff:
@@ -310,7 +310,6 @@ async fn test_verify_new_epoch_block_uncle_should_none_failed() {
}
#[stest::test]
-#[allow(clippy::vec_init_then_push)]
async fn test_verify_can_not_be_uncle_is_member_failed() {
let times = 5;
let (mut writeable_block_chain_service, node_config, storage) =
@@ -327,8 +326,7 @@ async fn test_verify_can_not_be_uncle_is_member_failed() {
.get_header_by_number(times - 2)
.unwrap()
.unwrap();
- let mut uncles = Vec::new();
- uncles.push(uncle_header);
+ let uncles = vec![uncle_header];
let apply_failed = apply_with_illegal_uncle(
node_config.net(),
uncles,
@@ -342,7 +340,6 @@ async fn test_verify_can_not_be_uncle_is_member_failed() {
}
#[stest::test]
-#[allow(clippy::vec_init_then_push)]
async fn test_verify_can_not_be_uncle_check_ancestor_failed() {
// 1. chain
let times = 7;
@@ -379,8 +376,7 @@ async fn test_verify_can_not_be_uncle_check_ancestor_failed() {
// 3. new block
let uncle_header = new_branch.current_header();
- let mut uncles = Vec::new();
- uncles.push(uncle_header);
+ let uncles = vec![uncle_header];
let apply_failed = apply_with_illegal_uncle(
node_config.net(),
uncles,
@@ -393,7 +389,6 @@ async fn test_verify_can_not_be_uncle_check_ancestor_failed() {
}
}
-#[allow(clippy::vec_init_then_push)]
async fn test_verify_illegal_uncle_future_timestamp(succ: bool) -> Result<Block> {
let count = 5;
let (mut uncle_header, mut writeable_block_chain_service, node_config, storage) =
@@ -404,8 +399,7 @@ async fn test_verify_illegal_uncle_future_timestamp(succ: bool) -> Result<Block>
.with_timestamp((duration_since_epoch().as_millis() + 1000) as u64)
.build();
}
- let mut uncles = Vec::new();
- uncles.push(uncle_header);
+ let uncles = vec![uncle_header];
apply_with_illegal_uncle(
node_config.net(),
uncles,
@@ -426,7 +420,6 @@ async fn test_verify_illegal_uncle_future_timestamp_failed() {
}
}
-#[allow(clippy::vec_init_then_push)]
async fn test_verify_illegal_uncle_consensus(succ: bool) -> Result<()> {
let mut genesis_config = BuiltinNetworkID::Test.genesis_config().clone();
genesis_config.genesis_block_parameter =
@@ -465,8 +458,7 @@ async fn test_verify_illegal_uncle_consensus(succ: bool) -> Result<()> {
}
// 3. main and create a new block with uncle block
- let mut uncles = Vec::new();
- uncles.push(uncle_block_header);
+ let uncles = vec![uncle_block_header];
let mut main_block_chain = mock_chain.fork_new_branch(None).unwrap();
let (block_template, _) = main_block_chain
.create_block_template(*miner.address(), None, Vec::new(), uncles, None)
@@ -533,12 +525,10 @@ async fn test_verify_block_used_gas_failed() {
}
#[stest::test]
-#[allow(clippy::vec_init_then_push)]
async fn test_verify_txn_count_failed() {
// TODO: fix me
let (mut new_block, mut main) = new_block_and_main().await;
- let mut txns = Vec::new();
- txns.push(SignedUserTransaction::mock());
+ let txns = vec![SignedUserTransaction::mock()];
let mut body = new_block.body.clone();
body.transactions = txns;
new_block.body = body;
@@ -659,7 +649,6 @@ async fn test_verify_uncles_count_failed() {
debug!("expect apply failed : {:?}", apply_failed.err().unwrap());
}
-#[allow(clippy::vec_init_then_push)]
async fn test_verify_uncles_number(succ: bool) -> Result<Block> {
let count = 5;
let (mut uncle_header, mut writeable_block_chain_service, node_config, storage) =
@@ -676,8 +665,7 @@ async fn test_verify_uncles_number(succ: bool) -> Result<Block> {
)
.build();
}
- let mut uncles = Vec::new();
- uncles.push(uncle_header);
+ let uncles = vec![uncle_header];
apply_with_illegal_uncle(
node_config.net(),
uncles,
@@ -696,7 +684,6 @@ async fn test_verify_uncles_number_failed() {
}
}
-#[allow(clippy::vec_init_then_push)]
async fn test_verify_uncles_in_old_epoch(begin_epoch: bool) -> Result<Block> {
let count = 5;
let (uncle_header, mut writeable_block_chain_service, node_config, storage) =
@@ -731,8 +718,7 @@ async fn test_verify_uncles_in_old_epoch(begin_epoch: bool) -> Result<Block> {
}
}
- let mut uncles = Vec::new();
- uncles.push(uncle_header);
+ let uncles = vec![uncle_header];
apply_with_illegal_uncle(
node_config.net(),
uncles,
@@ -751,14 +737,12 @@ async fn test_verify_uncles_in_old_epoch_failed() {
}
#[stest::test]
-#[allow(clippy::vec_init_then_push)]
async fn test_verify_uncles_uncle_exist_failed() {
let count = 5;
let (uncle_header, mut writeable_block_chain_service, node_config, storage) =
uncle_block_and_writeable_block_chain(count, count - 2).await;
let net = node_config.net();
- let mut uncles = Vec::new();
- uncles.push(uncle_header);
+ let uncles = vec![uncle_header];
info!(
"number 1 : {}",
writeable_block_chain_service
@@ -828,7 +812,6 @@ async fn test_some_uncles_in_block_failed() {
}
#[stest::test]
-#[allow(clippy::vec_init_then_push)]
async fn test_verify_uncle_and_parent_number_failed() {
let count = 5;
let (mut uncle_header, mut writeable_block_chain_service, node_config, storage) =
@@ -863,8 +846,7 @@ async fn test_verify_uncle_and_parent_number_failed() {
.number();
assert_eq!(old_number + 1, new_number);
- let mut uncles = Vec::new();
- uncles.push(uncle_header);
+ let uncles = vec![uncle_header];
let apply_failed = apply_with_illegal_uncle(
node_config.net(),
@@ -879,7 +861,6 @@ async fn test_verify_uncle_and_parent_number_failed() {
}
#[stest::test]
-#[allow(clippy::vec_init_then_push)]
async fn test_verify_uncle_which_parent_is_end_block_in_last_epoch() {
let count = TEST_CONFIG.consensus_config.epoch_block_count;
let (uncle_header, mut writeable_block_chain_service, node_config, storage) =
@@ -905,8 +886,7 @@ async fn test_verify_uncle_which_parent_is_end_block_in_last_epoch() {
(uncle_parent_header.number() + 1)
);
- let mut uncles = Vec::new();
- uncles.push(uncle_header);
+ let uncles = vec![uncle_header];
let apply_failed = apply_with_illegal_uncle(
node_config.net(),
uncles,

diff:
@@ -60,13 +60,12 @@ impl ConfigID {
}
}
-#[allow(clippy::vec_init_then_push)]
pub static ON_CHAIN_CONFIG_REGISTRY: Lazy<Vec<ConfigID>> = Lazy::new(|| {
- let mut configs: Vec<ConfigID> = Vec::new();
- configs.push(Version::config_id());
- configs.push(ConsensusConfig::config_id());
- configs.push(DaoConfig::config_id());
- configs
+ vec![
+ Version::config_id(),
+ ConsensusConfig::config_id(),
+ DaoConfig::config_id(),
+ ]
});
#[derive(Clone, Debug, PartialEq)]

original_message: remove clippy_vec_init_and_push and dup code (#3311)


message:
[dev] improve Package Manager detection mechenism
improve Package Manager detection on linux in scripts/dev_setup.sh
port from

diff:
@@ -822,8 +822,26 @@ if [ "$(whoami)" != 'root' ]; then
PRE_COMMAND=(sudo)
fi
-PACKAGE_MANAGER=
if [[ "$(uname)" == "Linux" ]]; then
+ # check for default package manager for linux
+ if [[ -f /etc/redhat-release ]]; then
+ # use yum by default
+ if command -v yum &>/dev/null; then
+ PACKAGE_MANAGER="yum"
+ elif command -v dnf &>/dev/null; then
+ # dnf is the updated default since Red Hat Enterprise Linux 8, CentOS 8, Fedora 22, and any distros based on these
+ echo "WARNING: dnf package manager support is experimental"
+ PACKAGE_MANAGER="dnf"
+ fi
+ elif [[ -f /etc/debian_version ]] && command -v apt-get &>/dev/null; then
+ PACKAGE_MANAGER="apt-get"
+ elif [[ -f /etc/arch-release ]] && command -v pacman &>/dev/null; then
+ PACKAGE_MANAGER="pacman"
+ elif [[ -f /etc/alpine-release ]] && command -v apk &>/dev/null; then
+ PACKAGE_MANAGER="apk"
+ fi
+ # if no default PACKAGE_MANAGER detected, pick one that's installed, this is usually useless
+ if [[ $PACKAGE_MANAGER == "" ]]; then
if command -v yum &>/dev/null; then
PACKAGE_MANAGER="yum"
elif command -v apt-get &>/dev/null; then
@@ -839,6 +857,7 @@ if [[ "$(uname)" == "Linux" ]]; then
echo "Unable to find supported package manager (yum, apt-get, dnf, or pacman). Abort"
exit 1
fi
+ fi
elif [[ "$(uname)" == "Darwin" ]]; then
if command -v brew &>/dev/null; then
PACKAGE_MANAGER="brew"

original_message:
[dev] improve Package Manager detection mechenism (#3412)

improve Package Manager detection on linux in scripts/dev_setup.sh
port from https://github.com/move-language/move/pull/146


message: add crypto_hash Blob example use sha3

diff:
@@ -2490,6 +2490,7 @@ name = "forkable-jellyfish-merkle"
version = "1.11.9"
dependencies = [
"anyhow",
+ "bcs",
"bcs-ext",
"byteorder 1.4.3",
"criterion",

diff:
@@ -29,6 +29,7 @@ proptest = "1.0.0"
proptest-derive = "0.3.0"
criterion = "0.3"
starcoin-crypto = { git = "https://github.com/starcoinorg/starcoin-crypto", rev = "d871dfb4216f034ee334a575926c101574d9d6dc"}
+bcs = "0.1.3"
[features]
default = []

diff:
@@ -982,3 +982,28 @@ where
.next()
.map(|(k, _v)| k.clone())
}
+
+#[test]
+fn blob_crypto_hash_test() -> Result<()> {
+ let buf = hex::decode(
+ "0xfa000000000000007b161ceeef010000000000000000000000000000000000000000000000000000"
+ .strip_prefix("0x")
+ .ok_or_else(|| format_err!("strip_prefix error"))?,
+ )?;
+ let blob = Blob::from(buf);
+ let hash = blob.crypto_hash();
+
+ let name = starcoin_crypto::_serde_name::trace_name::<Blob>()
+ .expect("The `CryptoHasher` macro only applies to structs and enums");
+ assert_eq!(name, "Blob");
+ let salt_prefix: &[u8] = b"STARCOIN::Blob";
+ let ser = bcs::to_bytes(&blob)?;
+ let salt = [
+ HashValue::sha3_256_of(salt_prefix).as_slice(),
+ ser.as_slice(),
+ ]
+ .concat();
+ let hash1 = HashValue::sha3_256_of(&salt[..]);
+ assert_eq!(hash, hash1);
+ Ok(())
+}

original_message: add crypto_hash Blob example use sha3 (#3416)


message:
fix import  snapshot bug
* add export_snapshot special block number
* 1.add export_snapshot special block number for debug
2.fix apply_snapshot on barnard
* remove unused code

diff:
@@ -346,6 +346,9 @@ pub struct ExportSnapshotOptions {
#[clap(long, short = 't')]
/// enable increment export snapshot
pub increment: Option<bool>,
+ #[clap(long, short = 'b')]
+ /// special block_num for debug usage
+ pub special_block_num: Option<BlockNumber>,
}
#[derive(Debug, Parser)]
@@ -479,7 +482,13 @@ fn main() -> anyhow::Result<()> {
}
if let Cmd::ExportSnapshot(option) = cmd {
- let result = export_snapshot(option.db_path, option.output, option.net, option.increment);
+ let result = export_snapshot(
+ option.db_path,
+ option.output,
+ option.net,
+ option.increment,
+ option.special_block_num,
+ );
return result;
}
@@ -1038,6 +1047,7 @@ pub fn export_snapshot(
output: PathBuf,
network: BuiltinNetworkID,
increment: Option<bool>,
+ special_block_num: Option<BlockNumber>,
) -> anyhow::Result<()> {
let start_time = SystemTime::now();
let net = ChainNetwork::new_builtin(network);
@@ -1064,11 +1074,16 @@ pub fn export_snapshot(
)
.expect("create block chain should success.");
let block_num = chain.status().head().number();
- let cur_num = if block_num <= SNAP_GAP {
+ let mut cur_num = if block_num <= SNAP_GAP {
block_num
} else {
block_num - SNAP_GAP
};
+ if let Some(special_num) = special_block_num {
+ if special_num <= cur_num {
+ cur_num = special_num;
+ }
+ }
let cur_block = chain
.get_block_by_number(cur_num)?
.ok_or_else(|| format_err!("get block by number {} error", cur_num))?;

diff:
@@ -458,45 +458,21 @@ impl ChainStateWriter for ChainStateDB {
fn apply(&self, chain_state_set: ChainStateSet) -> Result<()> {
for (address, account_state_set) in chain_state_set.state_sets() {
- let (code_root, resource_root) = match self.get_account_state(address)? {
- Some(account_state) => (
- account_state.code_root(),
- Some(account_state.resource_root()),
- ),
- None => (None, None),
- };
- let code_root = match (code_root, account_state_set.code_set()) {
- (Some(storage_root), Some(state_set)) => {
- let state_tree = self.new_state_tree::<ModuleName>(storage_root);
- state_tree.apply(state_set.clone())?;
- state_tree.flush()?;
- Some(state_tree.root_hash())
- }
- (Some(storage_root), None) => Some(storage_root),
- (None, Some(state_set)) => {
+ let code_root = if let Some(state_set) = account_state_set.code_set() {
let state_tree = StateTree::<ModuleName>::new(self.store.clone(), None);
state_tree.apply(state_set.clone())?;
state_tree.flush()?;
Some(state_tree.root_hash())
- }
- (None, None) => None,
+ } else {
+ None
};
-
- let resource_root = match (resource_root, account_state_set.resource_set()) {
- (Some(storage_root), Some(state_set)) => {
- let state_tree = self.new_state_tree::<StructTag>(storage_root);
- state_tree.apply(state_set.clone())?;
- state_tree.flush()?;
- state_tree.root_hash()
- }
- (Some(storage_root), None) => storage_root,
- (None, Some(state_set)) => {
+ let resource_root = if let Some(state_set) = account_state_set.resource_set() {
let state_tree = StateTree::<StructTag>::new(self.store.clone(), None);
state_tree.apply(state_set.clone())?;
state_tree.flush()?;
state_tree.root_hash()
- }
- (None, None) => unreachable!("this should never happened"),
+ } else {
+ unreachable!("this should never happened")
};
let new_account_state = AccountState::new(code_root, resource_root);
self.state_tree.put(*address, new_account_state.try_into()?);

original_message:
fix import  snapshot bug (#3425)

* add export_snapshot special block number

* 1.add export_snapshot special block number for debug
2.fix apply_snapshot on barnard

* remove unused code


message: fix some dependabot alter

diff:
@@ -278,9 +278,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
[[package]]
name = "ascii"
-version = "0.8.7"
+version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97be891acc47ca214468e09425d02cef3af2c94d0d82081cd02061f996802f14"
+checksum = "bbf56136a5198c7b01a49e3afcbef6cf84597273d298f54432926024107b0109"
[[package]]
name = "asn1_der"
@@ -1063,9 +1063,9 @@ dependencies = [
[[package]]
name = "chunked_transfer"
-version = "0.3.1"
+version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "498d20a7aaf62625b9bf26e637cf7736417cde1d0c99f1d04d1170229a85cf87"
+checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e"
[[package]]
name = "cipher"
@@ -7874,9 +7874,9 @@ dependencies = [
[[package]]
name = "rust-embed"
-version = "5.9.0"
+version = "6.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fe1fe6aac5d6bb9e1ffd81002340363272a7648234ec7bdfac5ee202cb65523"
+checksum = "9a17e5ac65b318f397182ae94e532da0ba56b88dd1200b774715d36c4943b1c3"
dependencies = [
"rust-embed-impl",
"rust-embed-utils",
@@ -7885,9 +7885,9 @@ dependencies = [
[[package]]
name = "rust-embed-impl"
-version = "5.9.0"
+version = "6.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ed91c41c42ef7bf687384439c312e75e0da9c149b0390889b94de3c7d9d9e66"
+checksum = "94e763e24ba2bf0c72bc6be883f967f794a019fafd1b86ba1daff9c91a7edd30"
dependencies = [
"proc-macro2 1.0.36",
"quote 1.0.17",
@@ -7898,10 +7898,11 @@ dependencies = [
[[package]]
name = "rust-embed-utils"
-version = "5.1.0"
+version = "7.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a512219132473ab0a77b52077059f1c47ce4af7fbdc94503e9862a34422876d"
+checksum = "756feca3afcbb1487a1d01f4ecd94cf8ec98ea074c55a69e7136d29fb6166029"
dependencies = [
+ "sha2 0.9.9",
"walkdir",
]
@@ -11280,15 +11281,15 @@ dependencies = [
[[package]]
name = "tiny_http"
-version = "0.6.4"
+version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2e22cb179b63e5fc2d0b5be237dc107da072e2407809ac70a8ce85b93fe8f562"
+checksum = "9ce51b50006056f590c9b7c3808c3bd70f0d1101666629713866c227d6e58d39"
dependencies = [
"ascii",
"chrono",
"chunked_transfer",
"log 0.4.16",
- "url 1.7.2",
+ "url 2.2.2",
]
[[package]]

diff:
@@ -8,14 +8,14 @@ edition = "2021"
[dependencies]
anyhow = "1.0.41"
-tiny_http = "0.6"
+tiny_http = "0.8.2"
futures = "0.3.12"
hex = { version = "0.4.3", default-features = false }
tokio = {version = "^1", features = ["full"] }
tokio-executor = { version = "0.2.0-alpha.6", features = ["blocking"] }
futures-timer = "3.0"
-ascii = "0.8"
-rust-embed = "5.9.0"
+ascii = "1.0.0"
+rust-embed = "6.3.0"
serde = { version = "1.0.130" }
serde_json = { version="1.0", features = ["arbitrary_precision"]}
once_cell = "1.12.0"

diff:
@@ -22,7 +22,7 @@ struct Asset;
fn index_html() -> String {
let index_html = Asset::get("index.html").unwrap();
- std::str::from_utf8(index_html.as_ref())
+ std::str::from_utf8(index_html.data.as_ref())
.unwrap()
.to_string()
}

diff:
@@ -10,7 +10,7 @@ description = "Starcoin metrics"
[dependencies]
anyhow = "1.0.41"
futures = "0.3.12"
-hyper = { version = "0.14", features = ["full"]}
+hyper = { version = "0.14.12", features = ["full"]}
starcoin-logger = { path = "../logger" }
prometheus = { version = "0.13.0", default-features = false, features =["push"] }
serde_json = { version="1.0", features = ["arbitrary_precision"]}

original_message: fix some dependabot alter (#3485)


message: [fix] import_snapshot dir

diff:
@@ -42,7 +42,7 @@ function import_snapshot() {
download "$net" "$from_dir"
- ./starcoin_db_exporter apply-snapshot -i "$from_dir" -n "$net" -o "$to_dir"
+ ./starcoin_db_exporter apply-snapshot -i "$from_dir"/snapshot -n "$net" -o "$to_dir"
case_status=$?
if [ $case_status -ne 0 ]; then
echo -e "apply-snapshot $net $from_dir fail"

original_message: [fix] import_snapshot dir (#3535)


message: bump indicatif 0.15->0.16.2, fix typo

diff:
@@ -1285,9 +1285,7 @@ dependencies = [
"encode_unicode",
"libc",
"once_cell",
- "regex",
"terminal_size",
- "unicode-width",
"winapi 0.3.9",
]
@@ -3484,9 +3482,9 @@ dependencies = [
[[package]]
name = "indicatif"
-version = "0.15.0"
+version = "0.16.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7baab56125e25686df467fe470785512329883aab42696d661247aca2a2896e4"
+checksum = "2d207dc617c7a380ab07ff572a6e52fa202a2a8f355860ac9c38e23f8196be1b"
dependencies = [
"console",
"lazy_static 1.4.0",
@@ -6235,9 +6233,9 @@ dependencies = [
[[package]]
name = "number_prefix"
-version = "0.3.0"
+version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
+checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "object"

diff:
@@ -28,7 +28,7 @@ serde = "~1"
serde_json = {version="~1", features=["arbitrary_precision"]}
anyhow="~1"
hex="~0.4"
-indicatif = "0.15.0"
+indicatif = "0.16.2"
logger = {path = "../../commons/logger",package="starcoin-logger"}
clap = { version = "3", features = ["derive"] }

diff:
@@ -509,7 +509,7 @@ pub fn export_block_range(
end: BlockNumber,
) -> anyhow::Result<()> {
let net = ChainNetwork::new_builtin(network);
- let db_stoarge = DBStorage::open_with_cfs(
+ let db_storage = DBStorage::open_with_cfs(
from_dir.join("starcoindb/db/starcoindb"),
StorageVersion::current_version()
.get_column_family_names()
@@ -520,7 +520,7 @@ pub fn export_block_range(
)?;
let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
CacheStorage::new(None),
- db_stoarge,
+ db_storage,
))?);
let (chain_info, _) =
Genesis::init_and_check_storage(&net, storage.clone(), from_dir.as_ref())?;
@@ -554,7 +554,7 @@ pub fn export_block_range(
.collect::<Vec<BlockNumber>>()
.into_iter()
.map(|num| {
- load_bar.set_message(format!("load block {}", num).as_str());
+ load_bar.set_message(format!("load block {}", num));
load_bar.inc(1);
chain
.get_block_by_number(num)?
@@ -572,7 +572,7 @@ pub fn export_block_range(
);
for block in block_list {
writeln!(file, "{}", serde_json::to_string(&block)?)?;
- bar.set_message(format!("write block {}", block.header().number()).as_str());
+ bar.set_message(format!("write block {}", block.header().number()));
bar.inc(1);
}
file.flush()?;
@@ -594,10 +594,10 @@ pub fn apply_block(
verifier: Verifier,
) -> anyhow::Result<()> {
let net = ChainNetwork::new_builtin(network);
- let db_stoarge = DBStorage::new(to_dir.join("starcoindb/db"), RocksdbConfig::default(), None)?;
+ let db_storage = DBStorage::new(to_dir.join("starcoindb/db"), RocksdbConfig::default(), None)?;
let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
CacheStorage::new(None),
- db_stoarge,
+ db_storage,
))?);
let (chain_info, _) = Genesis::init_and_check_storage(&net, storage.clone(), to_dir.as_ref())?;
let mut chain = BlockChain::new(
@@ -653,7 +653,7 @@ pub fn apply_block(
// apply block then flush startup_info for breakpoint resume
let startup_info = StartupInfo::new(block_hash);
storage.save_startup_info(startup_info)?;
- bar.set_message(format!("apply block {}", block_number).as_str());
+ bar.set_message(format!("apply block {}", block_number));
bar.inc(1);
}
bar.finish();
@@ -672,10 +672,10 @@ pub fn startup_info_back(
network: BuiltinNetworkID,
) -> anyhow::Result<()> {
let net = ChainNetwork::new_builtin(network);
- let db_stoarge = DBStorage::new(to_dir.join("starcoindb/db"), RocksdbConfig::default(), None)?;
+ let db_storage = DBStorage::new(to_dir.join("starcoindb/db"), RocksdbConfig::default(), None)?;
let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
CacheStorage::new(None),
- db_stoarge,
+ db_storage,
))?);
let (chain_info, _) = Genesis::init_and_check_storage(&net, storage.clone(), to_dir.as_ref())?;
let chain = BlockChain::new(
@@ -723,10 +723,10 @@ pub fn gen_block_transactions(
) -> anyhow::Result<()> {
::logger::init();
let net = ChainNetwork::new_builtin(BuiltinNetworkID::Halley);
- let db_stoarge = DBStorage::new(to_dir.join("starcoindb/db"), RocksdbConfig::default(), None)?;
+ let db_storage = DBStorage::new(to_dir.join("starcoindb/db"), RocksdbConfig::default(), None)?;
let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
CacheStorage::new(None),
- db_stoarge,
+ db_storage,
))?);
let (chain_info, _) = Genesis::init_and_check_storage(&net, storage.clone(), to_dir.as_ref())?;
let mut chain = BlockChain::new(
@@ -1028,7 +1028,7 @@ fn export_column(
}
};
start_index += max_size;
- bar.set_message(format!("export {} {}", column, index).as_str());
+ bar.set_message(format!("export {} {}", column, index));
bar.inc(1);
index += 1;
}
@@ -1052,7 +1052,7 @@ pub fn export_snapshot(
) -> anyhow::Result<()> {
let start_time = SystemTime::now();
let net = ChainNetwork::new_builtin(network);
- let db_stoarge = DBStorage::open_with_cfs(
+ let db_storage = DBStorage::open_with_cfs(
from_dir.join("starcoindb/db/starcoindb"),
StorageVersion::current_version()
.get_column_family_names()
@@ -1063,7 +1063,7 @@ pub fn export_snapshot(
)?;
let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
CacheStorage::new(None),
- db_stoarge,
+ db_storage,
))?);
let (chain_info, _) =
Genesis::init_and_check_storage(&net, storage.clone(), from_dir.as_ref())?;
@@ -1143,24 +1143,24 @@ pub fn export_snapshot(
.get_block_info(Some(block.id()))?
.ok_or_else(|| format_err!("get block info by hash {} error", block.id()))?;
let block_accumulator_info = block_info.get_block_accumulator_info();
- let mut mainfest_list = vec![];
+ let mut manifest_list = vec![];
let mut handles = Vec::with_capacity(5);
- mainfest_list.push((
+ manifest_list.push((
BLOCK_ACCUMULATOR_NODE_PREFIX_NAME,
cur_num,
block_accumulator_info.accumulator_root,
));
- mainfest_list.push((BLOCK_PREFIX_NAME, cur_num, block.header.id()));
- mainfest_list.push((BLOCK_INFO_PREFIX_NAME, cur_num, block.header.id()));
+ manifest_list.push((BLOCK_PREFIX_NAME, cur_num, block.header.id()));
+ manifest_list.push((BLOCK_INFO_PREFIX_NAME, cur_num, block.header.id()));
let txn_accumulator_info = block_info.get_txn_accumulator_info();
- mainfest_list.push((
+ manifest_list.push((
TRANSACTION_ACCUMULATOR_NODE_PREFIX_NAME,
txn_accumulator_info.get_num_leaves() - 1,
txn_accumulator_info.accumulator_root,
));
let mbar = MultiProgress::new();
- for (column, num_record, _hash) in mainfest_list.clone() {
+ for (column, num_record, _hash) in manifest_list.clone() {
let accumulator = match column {
BLOCK_ACCUMULATOR_NODE_PREFIX_NAME | BLOCK_PREFIX_NAME | BLOCK_INFO_PREFIX_NAME => {
MerkleAccumulator::new_with_info(
@@ -1214,7 +1214,7 @@ pub fn export_snapshot(
)?;
if index % BATCH_SIZE == 0 {
- bar.set_message(format!("export state {}", index / BATCH_SIZE).as_str());
+ bar.set_message(format!("export state {}", index / BATCH_SIZE));
bar.inc(1);
}
index += 1;
@@ -1230,7 +1230,7 @@ pub fn export_snapshot(
handle.join().unwrap().unwrap();
}
- mainfest_list.push((
+ manifest_list.push((
STATE_NODE_PREFIX_NAME,
nums.load(Ordering::Relaxed),
state_root,
@@ -1244,7 +1244,7 @@ pub fn export_snapshot(
// save manifest
let name_manifest = "manifest.csv".to_string();
let mut file_manifest = File::create(output.join(name_manifest))?;
- for (path, num, hash) in mainfest_list {
+ for (path, num, hash) in manifest_list {
writeln!(file_manifest, "{} {} {}", path, num, hash)?;
}
file_manifest.flush()?;
@@ -1297,7 +1297,7 @@ fn import_column(
}
}
if index % BATCH_SIZE == 0 {
- bar.set_message(format!("import {} {}", column, index / BATCH_SIZE).as_str());
+ bar.set_message(format!("import {} {}", column, index / BATCH_SIZE));
bar.inc(1);
}
index += 1;
@@ -1347,10 +1347,10 @@ pub fn apply_snapshot(
) -> anyhow::Result<()> {
let start_time = SystemTime::now();
let net = ChainNetwork::new_builtin(network);
- let db_stoarge = DBStorage::new(to_dir.join("starcoindb/db"), RocksdbConfig::default(), None)?;
+ let db_storage = DBStorage::new(to_dir.join("starcoindb/db"), RocksdbConfig::default(), None)?;
let storage = Arc::new(Storage::new(StorageInstance::new_cache_and_db_instance(
CacheStorage::new(None),
- db_stoarge,
+ db_storage,
))?);
let (chain_info, _) = Genesis::init_and_check_storage(&net, storage.clone(), to_dir.as_ref())?;
let mut chain = BlockChain::new(
@@ -1433,14 +1433,11 @@ pub fn apply_snapshot(
account_states.push((account_address, account_state_set));
index += 1;
if index % BATCH_SIZE == 0 {
- bar.set_message(
- format!(
+ bar.set_message(format!(
"import {} index {}",
STATE_NODE_PREFIX_NAME,
index / BATCH_SIZE
- )
- .as_str(),
- );
+ ));
bar.inc(1);
}
}

diff:
@@ -15,7 +15,7 @@ use std::collections::HashMap;
use std::sync::Arc;
pub struct AccumulatorTree {
- /// forzen subtree roots hashes.
+ /// frozen subtree roots hashes.
frozen_subtree_roots: Vec<HashValue>,
/// The total number of leaves in this accumulator.
pub(crate) num_leaves: LeafCount,
@@ -163,7 +163,7 @@ impl AccumulatorTree {
//aggregator all nodes
not_frozen_nodes.extend_from_slice(&to_freeze);
self.update_temp_nodes(not_frozen_nodes.clone());
- // udpate to cache
+ // update to cache
self.update_cache(not_frozen_nodes);
// update self properties
self.root_hash = hash;

original_message: bump indicatif 0.15->0.16.2, fix typo (#3544)


diff:
@@ -2709,12 +2709,6 @@ dependencies = [
"slab",
]
-[[package]]
-name = "gcc"
-version = "0.3.55"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
-
[[package]]
name = "generic-array"
version = "0.12.4"
@@ -4777,20 +4771,16 @@ dependencies = [
[[package]]
name = "merkletree"
-version = "0.21.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2202d6eb93a775e0a619fc08924b341d0cf12d6e0f064ee52d24311de45b925b"
+version = "0.22.0"
+source = "git+https://github.com/filecoin-project/merkletree?rev=05dd6de235e0dc36dbe05cb0849f35957fdca5be#05dd6de235e0dc36dbe05cb0849f35957fdca5be"
dependencies = [
"anyhow",
"arrayref",
"log 0.4.17",
"memmap",
"positioned-io",
- "rand 0.7.3",
"rayon",
- "rust-crypto",
"serde 1.0.140",
- "tempdir",
"tempfile",
"typenum",
]
@@ -7312,29 +7302,6 @@ dependencies = [
"nibble_vec",
]
-[[package]]
-name = "rand"
-version = "0.3.23"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64ac302d8f83c0c1974bf758f6b041c6c8ada916fbb44a609158ca8b064cc76c"
-dependencies = [
- "libc",
- "rand 0.4.6",
-]
-
-[[package]]
-name = "rand"
-version = "0.4.6"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
-dependencies = [
- "fuchsia-cprng",
- "libc",
- "rand_core 0.3.1",
- "rdrand",
- "winapi 0.3.9",
-]
-
[[package]]
name = "rand"
version = "0.6.5"
@@ -7895,19 +7862,6 @@ dependencies = [
"crossbeam-utils 0.8.11",
]
-[[package]]
-name = "rust-crypto"
-version = "0.2.36"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f76d05d3993fd5f4af9434e8e436db163a12a9d40e1a58a726f27a01dfd12a2a"
-dependencies = [
- "gcc",
- "libc",
- "rand 0.3.23",
- "rustc-serialize",
- "time 0.1.44",
-]
-
[[package]]
name = "rust-embed"
version = "6.4.0"
@@ -11050,16 +11004,6 @@ dependencies = [
"target-lexicon",
]
-[[package]]
-name = "tempdir"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
-dependencies = [
- "rand 0.4.6",
- "remove_dir_all",
-]
-
[[package]]
name = "tempfile"
version = "3.3.0"

diff:
@@ -14,7 +14,7 @@ serde_json = { version = "~1", features = ["arbitrary_precision"] }
anyhow = "~1"
hex = "~0.4"
bcs-ext = { package = "bcs-ext", path = "../../commons/bcs_ext" }
-merkletree = { version = "0.21.0", features = ["chaincore"] }
+merkletree = { git = "https://github.com/filecoin-project/merkletree", rev = "05dd6de235e0dc36dbe05cb0849f35957fdca5be" }
starcoin-vm-types = { path = "../../vm/types" }
starcoin-crypto = { git = "https://github.com/starcoinorg/starcoin-crypto", rev = "d871dfb4216f034ee334a575926c101574d9d6dc" }
clap = { version = "3", features = ["derive"] }

original_message: update dependency merkletree remove rust-crypto (#3632)


diff:
@@ -28,35 +28,35 @@ module RLP {
/// Nested arrays are not supported.
public fun decode_list(data: &vector<u8>): vector<vector<u8>> {
let (decoded, consumed) = decode(data, 0);
- assert(consumed == Vector::length(data), INVALID_RLP_DATA);
+ assert!(consumed == Vector::length(data), INVALID_RLP_DATA);
decoded
}
fun decode(data: &vector<u8>, offset: u64): (vector<vector<u8>>, u64) {
let data_len = Vector::length(data);
- assert(offset < data_len, DATA_TOO_SHORT);
+ assert!(offset < data_len, DATA_TOO_SHORT);
let first_byte = *Vector::borrow(data, offset);
if (first_byte >= 248u8) { // 0xf8
let length_of_length = ((first_byte - 247u8) as u64);
- assert(offset + length_of_length < data_len, DATA_TOO_SHORT);
+ assert!(offset + length_of_length < data_len, DATA_TOO_SHORT);
let length = unarrayify_integer(data, offset + 1, (length_of_length as u8));
- assert(offset + length_of_length + length < data_len, DATA_TOO_SHORT);
+ assert!(offset + length_of_length + length < data_len, DATA_TOO_SHORT);
decode_children(data, offset, offset + 1 + length_of_length, length_of_length + length)
} else if (first_byte >= 192u8) { // 0xc0
let length = ((first_byte - 192u8) as u64);
- assert(offset + length < data_len, DATA_TOO_SHORT);
+ assert!(offset + length < data_len, DATA_TOO_SHORT);
decode_children(data, offset, offset + 1, length)
} else if (first_byte >= 184u8) { // 0xb8
let length_of_length = ((first_byte - 183u8) as u64);
- assert(offset + length_of_length < data_len, DATA_TOO_SHORT);
+ assert!(offset + length_of_length < data_len, DATA_TOO_SHORT);
let length = unarrayify_integer(data, offset + 1, (length_of_length as u8));
- assert(offset + length_of_length + length < data_len, DATA_TOO_SHORT);
+ assert!(offset + length_of_length + length < data_len, DATA_TOO_SHORT);
let bytes = Bytes::slice(data, offset + 1 + length_of_length, offset + 1 + length_of_length + length);
(Vector::singleton(bytes), 1+length_of_length+length)
} else if (first_byte >= 128u8) { // 0x80
let length = ((first_byte - 128u8) as u64);
- assert(offset + length < data_len, DATA_TOO_SHORT);
+ assert!(offset + length < data_len, DATA_TOO_SHORT);
let bytes = Bytes::slice(data, offset + 1, offset + 1 + length);
(Vector::singleton(bytes), 1+length)
} else {
@@ -72,7 +72,7 @@ module RLP {
let (decoded, consumed) = decode(data, child_offset);
Vector::append(&mut result, decoded);
child_offset = child_offset + consumed;
- assert(child_offset <= offset + 1 + length, DATA_TOO_SHORT);
+ assert!(child_offset <= offset + 1 + length, DATA_TOO_SHORT);
};
(result, 1 + length)
}

diff:
@@ -115,11 +115,11 @@ module MerkleDistributor {
fun internal_claim<T: store>(distribution: &mut MerkleDistribution<T>, index: u64, account: address, amount: u128, merkle_proof: vector<vector<u8>>): Token<T> {
let claimed = is_claimed_(distribution, index);
- assert(!claimed, Errors::custom(ALREADY_CLAIMED));
+ assert!(!claimed, Errors::custom(ALREADY_CLAIMED));
let leaf_data = encode_leaf(&index, &account, &amount);
let verified = MerkleProof::verify(&merkle_proof, &distribution.merkle_root, Hash::sha3_256(leaf_data));
- assert(verified, Errors::custom(INVALID_PROOF));
+ assert!(verified, Errors::custom(INVALID_PROOF));
set_claimed_(distribution, index);

original_message: [test] use replace assert with assert! in move (#3650)


message: [test] fix address in address_hash test

diff:
@@ -43,7 +43,7 @@ mod test {
#[test]
fn address_hash() {
- let address: AccountAddress = "ca843279e3427144cead5e4d5999a3d0".parse().unwrap();
+ let address: AccountAddress = "0xca843279e3427144cead5e4d5999a3d0".parse().unwrap();
let hash_vec =
Vec::from_hex("7d39654178dd4758d0bc33b26e3e06051f04a215fd7ad270d4fb5e4988c8e5d2")

original_message: [test] fix address in address_hash test (#3651)


message: [ci] make ci no-fail-fast

diff:
@@ -8,7 +8,7 @@ $env:RUSTC_BOOTSTRAP=1
$env:RUST_MIN_STACK=8*1024*1024
$env:RUST_LOG="OFF"
$env:RUST_BACKTRACE=0
-cargo xtest --exclude starcoin-move-prover -j 15 -- --test-threads=10 --color never --format pretty | Tee-Object "$TEST_RESULT_FILE"
+cargo xtest --no-fail-fast --exclude starcoin-move-prover -j 15 -- --test-threads=10 --color never --format pretty | Tee-Object "$TEST_RESULT_FILE"
$failed_tests=Select-String -Pattern 'test .* +\.\.\. FAILED' -Path "./target/debug/test_result.txt" -AllMatches
Write-Host "All failed tests are redirected to file: $TEST_RESULT_FAILED_FILE" -ForegroundColor Green

diff:
@@ -14,7 +14,7 @@ echo check ulimits
ulimit -a
#pleanse ensure tow test command's argument is same.
-RUST_LOG=OFF RUST_BACKTRACE=0 cargo xtest --exclude starcoin-move-prover -j 15 -- --test-threads=10 --color never --format pretty |tee "$TEST_RESULT_FILE" ||true
+RUST_LOG=OFF RUST_BACKTRACE=0 cargo xtest --no-fail-fast --exclude starcoin-move-prover -j 15 -- --test-threads=10 --color never --format pretty |tee "$TEST_RESULT_FILE" ||true
grep -e '^test[[:space:]][^[:space:]]*[[:space:]]\+\.\.\.[[:space:]]FAILED' "$TEST_RESULT_FILE" >"$TEST_RESULT_FAILED_FILE" ||true
status=0

original_message: [ci] make ci no-fail-fast (#3663)


message:
[dep][security] bump filecoin-project/merkletree, remove memmap
* [dep][security] bump filecoin-project/merkletree
use maintained memmap2
address
* update cargo.lock

diff:
@@ -4726,21 +4726,11 @@ version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
-[[package]]
-name = "memmap"
-version = "0.7.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
-dependencies = [
- "libc",
- "winapi 0.3.9",
-]
-
[[package]]
name = "memmap2"
-version = "0.5.5"
+version = "0.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a79b39c93a7a5a27eeaf9a23b5ff43f1b9e0ad6b1cdd441140ae53c35613fc7"
+checksum = "95af15f345b17af2efc8ead6080fb8bc376f8cec1b35277b935637595fe77498"
dependencies = [
"libc",
]
@@ -4774,12 +4764,12 @@ dependencies = [
[[package]]
name = "merkletree"
version = "0.22.0"
-source = "git+https://github.com/filecoin-project/merkletree?rev=05dd6de235e0dc36dbe05cb0849f35957fdca5be#05dd6de235e0dc36dbe05cb0849f35957fdca5be"
+source = "git+https://github.com/filecoin-project/merkletree?rev=3b1d98c43b341aed935152c5c1535c17d1b21d20#3b1d98c43b341aed935152c5c1535c17d1b21d20"
dependencies = [
"anyhow",
"arrayref",
"log 0.4.17",
- "memmap",
+ "memmap2",
"positioned-io",
"rayon",
"serde 1.0.140",

diff:
@@ -12,7 +12,7 @@ bcs-ext = {package = "bcs-ext", path = "../../commons/bcs_ext"}
clap = {version = "3", features = ["derive"]}
csv = "~1"
hex = "~0.4"
-merkletree = {git = "https://github.com/filecoin-project/merkletree", rev = "05dd6de235e0dc36dbe05cb0849f35957fdca5be"}
+merkletree = {git = "https://github.com/filecoin-project/merkletree", rev = "3b1d98c43b341aed935152c5c1535c17d1b21d20"}
serde = "~1"
serde_json = {version = "~1", features = ["arbitrary_precision"]}
sha3 = "0.9.1"

original_message:
[dep][security] bump filecoin-project/merkletree, remove memmap (#3671)

* [dep][security] bump filecoin-project/merkletree

use maintained memmap2
address https://github.com/starcoinorg/starcoin/issues/2403

* update cargo.lock


message: test with nextest

diff:
@@ -43,8 +43,10 @@ jobs:
with:
command: build
args: --all
+ - uses: taiki-e/install-action@nextest
- name: test
- run: bash ./scripts/auto_rerun_test.sh
+ # run: bash ./scripts/auto_rerun_test.sh
+ run: cargo nextest run --workspace --lib --retries 10 --build-jobs 10
# - name: check changed files
# run: bash ./scripts/changed_files.sh
- name: integration test dev environment

original_message: test with nextest (#3719)


diff:
@@ -289,7 +289,7 @@ pub async fn test_full_sync_fork() -> Result<()> {
Ok(())
}
-#[stest::test]
+#[stest::test(timeout = 120)]
pub async fn test_full_sync_fork_from_genesis() -> Result<()> {
let net1 = ChainNetwork::new_builtin(BuiltinNetworkID::Test);
let mut node1 = SyncNodeMocker::new(net1, 1, 50)?;
@@ -342,7 +342,7 @@ pub async fn test_full_sync_fork_from_genesis() -> Result<()> {
Ok(())
}
-#[stest::test]
+#[stest::test(timeout = 120)]
pub async fn test_full_sync_continue() -> Result<()> {
let net1 = ChainNetwork::new_builtin(BuiltinNetworkID::Test);
let mut node1 = SyncNodeMocker::new(net1, 10, 50)?;

original_message: [test] larger timeout on `full_sync` test (#3744)


message: 1.fix export snapshot

diff:
@@ -1153,6 +1153,8 @@ fn export_column(
/// block_info num block.header.hash
/// txn_accumulator num accumulator_root_hash
/// state num state_root_hash
+/// state_node_prev num state_root_hash
+
pub fn export_snapshot(
from_dir: PathBuf,
output: PathBuf,
@@ -1222,7 +1224,7 @@ pub fn export_snapshot(
let num = str_list[1].parse::<u64>()?;
old_snapshot_nums.insert(column, num);
}
- if old_snapshot_nums.len() != 5 {
+ if old_snapshot_nums.len() != 6 {
println!("increment export snapshot manifest.cvs error");
std::process::exit(1);
}
@@ -1337,27 +1339,27 @@ pub fn export_snapshot(
}
manifest_list.push((
- STATE_NODE_PREFIX_NAME_PREV,
- nums_prev.load(Ordering::Relaxed),
- state_root_prev,
+ STATE_NODE_PREFIX_NAME,
+ nums.load(Ordering::Relaxed),
+ state_root,
));
println!(
"{} nums {}",
- STATE_NODE_PREFIX_NAME_PREV,
- nums_prev.load(Ordering::Relaxed)
+ STATE_NODE_PREFIX_NAME,
+ nums.load(Ordering::Relaxed)
);
manifest_list.push((
- STATE_NODE_PREFIX_NAME,
- nums.load(Ordering::Relaxed),
- state_root,
+ STATE_NODE_PREFIX_NAME_PREV,
+ nums_prev.load(Ordering::Relaxed),
+ state_root_prev,
));
println!(
"{} nums {}",
- STATE_NODE_PREFIX_NAME,
- nums.load(Ordering::Relaxed)
+ STATE_NODE_PREFIX_NAME_PREV,
+ nums_prev.load(Ordering::Relaxed)
);
// save manifest
@@ -1654,7 +1656,7 @@ pub fn apply_snapshot(
if index % BATCH_SIZE == 0 {
bar.set_message(format!(
"import {} index {}",
- STATE_NODE_PREFIX_NAME,
+ STATE_NODE_PREFIX_NAME_PREV,
index / BATCH_SIZE
));
bar.inc(1);

original_message: 1.fix export snapshot (#3759)


diff:
#endif
// ANDROID, FreeBSD, OpenBSD and NetBSD also don't need timeb.h
+// use sys/timeb.h for glibc <= 2.31, or MSVC
#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) \
&& !defined(__NetBSD__) \
- && (defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 31)
+ && ((defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 31) || defined(_MSC_VER))
#include <sys/timeb.h>
#else
// use gettimeofday to get rid of `ftime` warning if glibc >= 2.31
@@ -509,7 +510,7 @@ static void oaes_get_seed( char buf[RANDSIZ + 1] )
static uint32_t oaes_get_seed(void)
{
#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__ANDROID__) && !defined(__NetBSD__) \
- && (defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 31)
+ && ((defined(__GLIBC__) && __GLIBC__ <= 2 && __GLIBC_MINOR__ < 31) || defined(_MSC_VER))
struct timeb timer;
struct tm *gmTimer;
char * _test = NULL;

original_message: fix compilation issue of cyptonight on windows (#3762)


message: redis protocol compatible

diff:
@@ -47,7 +47,7 @@ void GeoAddCmd::DoInitial(PikaCmdArgsType &argv, const CmdInfo* const ptr_info)
void GeoAddCmd::Do() {
nemo::Status s;
- int64_t response = 0, ret;
+ int64_t count = 0, ret;
bool exist = false;
const std::shared_ptr<nemo::Nemo> db = g_pika_server->db();
std::vector<GeoPoint>::const_iterator iter = pos_.begin();
@@ -68,16 +68,16 @@ void GeoAddCmd::Do() {
}
s = db->ZAdd(key_, score, iter->name, &ret);
if (s.ok() && !exist) {
- response = 1;
+ count += 1;
} else if (s.ok() && exist) {
- response = 0;
+
} else {
res_.SetRes(CmdRes::kErrOther, s.ToString());
return;
}
}
SlotKeyAdd("z", key_);
- res_.AppendInteger(response);
+ res_.AppendInteger(count);
return;
}

license: BSD 3-Clause New or Revised License

original_message: redis protocol compatible


message: Support GEORADIUS AND GEORADIUSBYMEMBER

diff:
@@ -166,16 +166,18 @@ const std::string kCmdNameGeoAdd = "geoadd";
const std::string kCmdNameGeoPos = "geopos";
const std::string kCmdNameGeoDist = "geodist";
const std::string kCmdNameGeoHash = "geohash";
+const std::string kCmdNameGeoRadius = "georadius";
+const std::string kCmdNameGeoRadiusByMember = "georadiusbymember";
typedef pink::RedisCmdArgsType PikaCmdArgsType;
enum CmdFlagsMask {
kCmdFlagsMaskRW = 1,
- kCmdFlagsMaskType = 14,
- kCmdFlagsMaskLocal = 16,
- kCmdFlagsMaskSuspend = 32,
- kCmdFlagsMaskPrior = 64,
- kCmdFlagsMaskAdminRequire = 128
+ kCmdFlagsMaskType = 28,
+ kCmdFlagsMaskLocal = 32,
+ kCmdFlagsMaskSuspend = 64,
+ kCmdFlagsMaskPrior = 128,
+ kCmdFlagsMaskAdminRequire = 255
};
enum CmdFlags {
@@ -189,15 +191,15 @@ enum CmdFlags {
kCmdFlagsZset = 10,
kCmdFlagsBit = 12,
kCmdFlagsHyperLogLog = 14,
+ kCmdFlagsGeo = 16,
kCmdFlagsNoLocal = 0, //default nolocal
- kCmdFlagsLocal = 16,
- kCmdFlagsGeo = 18,
+ kCmdFlagsLocal = 32,
kCmdFlagsNoSuspend = 0, //default nosuspend
- kCmdFlagsSuspend = 32,
+ kCmdFlagsSuspend = 64,
kCmdFlagsNoPrior = 0, //default noprior
- kCmdFlagsPrior = 64,
+ kCmdFlagsPrior = 128,
kCmdFlagsNoAdminRequire = 0, //default no need admin
- kCmdFlagsAdminRequire = 128
+ kCmdFlagsAdminRequire = 255
};

diff:
/*
* zset
*/
+ enum Sort
+{
+ Asc,
+ Desc
+};
struct GeoPoint {
- std::string name;
+ std::string member;
double longitude;
double latitude;
};
+ struct NeighborPoint{
+ std::string member;
+ double score;
+ double distance;
+ };
+
+ struct GeoRange {
+ std::string member;
+ double longitude;
+ double latitude;
+ double distance;
+ std::string unit;
+ bool withdist;
+ bool withhash;
+ bool withcoord;
+ int option_num;
+ bool count;
+ int count_limit;
+ Sort sort;
+ };
+
class GeoAddCmd : public Cmd {
public:
GeoAddCmd() {}
@@ -35,7 +61,7 @@ public:
virtual void Do();
private:
std::string key_;
- std::vector<std::string> name_;
+ std::vector<std::string> member_;
virtual void DoInitial(PikaCmdArgsType &argvs, const CmdInfo* const ptr_info);
};
@@ -54,8 +80,44 @@ public:
virtual void Do();
private:
std::string key_;
- std::vector<std::string> name_;
+ std::vector<std::string> member_;
+ virtual void DoInitial(PikaCmdArgsType &argvs, const CmdInfo* const ptr_info);
+};
+
+class GeoRadiusCmd : public Cmd {
+public:
+ GeoRadiusCmd() {}
+ virtual void Do();
+private:
+ std::string key_;
+ GeoRange range_;
+ virtual void DoInitial(PikaCmdArgsType &argvs, const CmdInfo* const ptr_info);
+ virtual void Clear() {
+ range_.withdist = false;
+ range_.withcoord = false;
+ range_.withhash = false;
+ range_.count = false;
+ range_.option_num = 0;
+ range_.count_limit = 0;
+ }
+};
+
+class GeoRadiusByMemberCmd : public Cmd {
+public:
+ GeoRadiusByMemberCmd() {}
+ virtual void Do();
+private:
+ std::string key_;
+ GeoRange range_;
virtual void DoInitial(PikaCmdArgsType &argvs, const CmdInfo* const ptr_info);
+ virtual void Clear() {
+ range_.withdist = false;
+ range_.withcoord = false;
+ range_.withhash = false;
+ range_.count = false;
+ range_.option_num = 0;
+ range_.count_limit = 0;
+ }
};
#endif

diff:
@@ -392,14 +392,20 @@ void InitCmdInfoTable() {
CmdInfo* geoaddptr = new CmdInfo(kCmdNameGeoAdd, -4, kCmdFlagsWrite | kCmdFlagsGeo);
cmd_infos.insert(std::pair<std::string, CmdInfo*>(kCmdNameGeoAdd, geoaddptr));
////GeoPos
- CmdInfo* geoposptr = new CmdInfo(kCmdNameGeoPos, -3, kCmdFlagsWrite | kCmdFlagsGeo);
+ CmdInfo* geoposptr = new CmdInfo(kCmdNameGeoPos, -3, kCmdFlagsRead | kCmdFlagsGeo);
cmd_infos.insert(std::pair<std::string, CmdInfo*>(kCmdNameGeoPos, geoposptr));
////GeoDist
CmdInfo* geodistptr = new CmdInfo(kCmdNameGeoDist, -3, kCmdFlagsWrite | kCmdFlagsGeo);
cmd_infos.insert(std::pair<std::string, CmdInfo*>(kCmdNameGeoDist, geodistptr));
////GeoHash
- CmdInfo* geohashptr = new CmdInfo(kCmdNameGeoHash, -3, kCmdFlagsWrite | kCmdFlagsGeo);
+ CmdInfo* geohashptr = new CmdInfo(kCmdNameGeoHash, -3, kCmdFlagsRead | kCmdFlagsGeo);
cmd_infos.insert(std::pair<std::string, CmdInfo*>(kCmdNameGeoHash, geohashptr));
+ ////GeoRadius
+ CmdInfo* georadiusptr = new CmdInfo(kCmdNameGeoRadius, -3, kCmdFlagsRead | kCmdFlagsGeo);
+ cmd_infos.insert(std::pair<std::string, CmdInfo*>(kCmdNameGeoRadius, georadiusptr));
+ ////GeoRadiusByMember
+ CmdInfo* georadiusbymemberptr = new CmdInfo(kCmdNameGeoRadiusByMember, -3, kCmdFlagsRead | kCmdFlagsGeo);
+ cmd_infos.insert(std::pair<std::string, CmdInfo*>(kCmdNameGeoRadiusByMember, georadiusbymemberptr));
}
void DestoryCmdInfoTable() {
@@ -795,6 +801,12 @@ void InitCmdTable(std::unordered_map<std::string, Cmd*> *cmd_table) {
////GeoHash
Cmd * geohashptr = new GeoHashCmd();
cmd_table->insert(std::pair<std::string, Cmd*>(kCmdNameGeoHash, geohashptr));
+ ////GeoRadius
+ Cmd * georadiusptr = new GeoRadiusCmd();
+ cmd_table->insert(std::pair<std::string, Cmd*>(kCmdNameGeoRadius, georadiusptr));
+ ////GeoRadiusByMember
+ Cmd * georadiusbymemberptr = new GeoRadiusByMemberCmd();
+ cmd_table->insert(std::pair<std::string, Cmd*>(kCmdNameGeoRadiusByMember, georadiusbymemberptr));
}
Cmd* GetCmdFromTable(const std::string& opt,

diff:
// of patent rights can be found in the PATENTS file in the same directory.
#include <sstream>
+#include <algorithm>
#include "slash_string.h"
#include "nemo.h"
#include "pika_geo.h"
@@ -37,7 +38,7 @@ void GeoAddCmd::DoInitial(PikaCmdArgsType &argv, const CmdInfo* const ptr_info)
res_.SetRes(CmdRes::kInvalidFloat);
return;
}
- point.name = argv[index+2];
+ point.member = argv[index+2];
point.longitude = longitude;
point.latitude = latitude;
pos_.push_back(point);
@@ -60,13 +61,13 @@ void GeoAddCmd::Do() {
std::string str_bits = std::to_string(bits);
double previous_score, score;
slash::string2d(str_bits.data(), str_bits.size(), &score);
- s = db->ZScore(key_, iter->name, &previous_score);
+ s = db->ZScore(key_, iter->member, &previous_score);
if (s.ok()) {
exist = true;
} else if (s.IsNotFound()){
exist = false;
}
- s = db->ZAdd(key_, score, iter->name, &ret);
+ s = db->ZAdd(key_, score, iter->member, &ret);
if (s.ok() && !exist) {
count += 1;
} else if (s.ok() && exist) {
@@ -87,17 +88,17 @@ void GeoPosCmd::DoInitial(PikaCmdArgsType &argv, const CmdInfo* const ptr_info)
return;
}
key_ = argv[1];
- name_.clear();
+ member_.clear();
size_t pos = 2;
while (pos < argv.size()) {
- name_.push_back(argv[pos++]);
+ member_.push_back(argv[pos++]);
}
}
void GeoPosCmd::Do() {
double score;
- res_.AppendArrayLen(name_.size());
- for (auto v : name_) {
+ res_.AppendArrayLen(member_.size());
+ for (auto v : member_) {
nemo::Status s = g_pika_server->db()->ZScore(key_, v, &score);
if (s.ok()) {
double xy[2];
@@ -125,6 +126,20 @@ void GeoPosCmd::Do() {
}
}
+static double length_converter(double meters, std::string unit) {
+ if (unit == "m") {
+ return meters;
+ } else if (unit == "km") {
+ return meters / 1000;
+ } else if (unit == "ft") {
+ return meters / 0.3048;
+ } else if (unit == "mi") {
+ return meters / 1609.34;
+ } else {
+ return -1;
+ }
+}
+
void GeoDistCmd::DoInitial(PikaCmdArgsType &argv, const CmdInfo* const ptr_info) {
if (!ptr_info->CheckArg(argv.size())) {
res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoDist);
@@ -169,15 +184,8 @@ void GeoDistCmd::Do() {
}
double distance = geohashGetDistance(first_xy[0], first_xy[1], second_xy[0], second_xy[1]);
- if (unit_ == "m") {
- distance = distance;
- } else if (unit_ == "km") {
- distance = distance / 1000;
- } else if (unit_ == "ft") {
- distance = distance / 0.3048;
- } else if (unit_ == "mi") {
- distance = distance / 1609.34;
- } else {
+ distance = length_converter(distance, unit_);
+ if (distance == -1) {
res_.SetRes(CmdRes::kErrOther, "unsupported unit provided. please use m, km, ft, mi");
return;
}
@@ -193,17 +201,17 @@ void GeoHashCmd::DoInitial(PikaCmdArgsType &argv, const CmdInfo* const ptr_info)
return;
}
key_ = argv[1];
- name_.clear();
+ member_.clear();
size_t pos = 2;
while (pos < argv.size()) {
- name_.push_back(argv[pos++]);
+ member_.push_back(argv[pos++]);
}
}
void GeoHashCmd::Do() {
- char * geoalphabet= "0123456789bcdefghjkmnpqrstuvwxyz";
- res_.AppendArrayLen(name_.size());
- for (auto v : name_) {
+ const char * geoalphabet= "0123456789bcdefghjkmnpqrstuvwxyz";
+ res_.AppendArrayLen(member_.size());
+ for (auto v : member_) {
double score;
nemo::Status s = g_pika_server->db()->ZScore(key_, v, &score);
if (s.ok()) {
@@ -237,3 +245,236 @@ void GeoHashCmd::Do() {
}
}
}
+
+static bool sort_distance_asc(const NeighborPoint & pos1, const NeighborPoint & pos2) {
+ return pos1.distance < pos2.distance;
+}
+
+static bool sort_distance_desc(const NeighborPoint & pos1, const NeighborPoint & pos2) {
+ return pos1.distance > pos2.distance;
+}
+
+static void GetAllNeighbors(std::string & key, GeoRange & range, CmdRes & res) {
+ nemo::Status s;
+ double longitude = range.longitude, latitude = range.latitude, distance = range.distance;
+ int count_limit = 0;
+
+ if (range.unit == "m") {
+ distance = distance;
+ } else if (range.unit == "km") {
+ distance = distance * 1000;
+ } else if (range.unit == "ft") {
+ distance = distance * 0.3048;
+ } else if (range.unit == "mi") {
+ distance = distance * 1609.34;
+ } else {
+ res.SetRes(CmdRes::kErrOther, "unsupported unit provided. please use m, km, ft, mi");
+ return;
+ }
+ // Search the zset for all matching points
+ GeoHashRadius georadius = geohashGetAreasByRadiusWGS84(longitude, latitude, distance);
+ GeoHashBits neighbors[9];
+ neighbors[0] = georadius.hash;
+ neighbors[1] = georadius.neighbors.north;
+ neighbors[2] = georadius.neighbors.south;
+ neighbors[3] = georadius.neighbors.east;
+ neighbors[4] = georadius.neighbors.west;
+ neighbors[5] = georadius.neighbors.north_east;
+ neighbors[6] = georadius.neighbors.north_west;
+ neighbors[7] = georadius.neighbors.south_east;
+ neighbors[8] = georadius.neighbors.south_west;
+
+ // For each neighbor (*and* our own hashbox), get all the matching
+ // members and add them to the potential result list.
+ std::vector<NeighborPoint> result;
+ for (size_t i = 0; i < sizeof(neighbors) / sizeof(*neighbors); i++) {
+ GeoHashFix52Bits min, max;
+ if (HASHISZERO(neighbors[i]))
+ continue;
+ min = geohashAlign52Bits(neighbors[i]);
+ neighbors[i].bits++;
+ max = geohashAlign52Bits(neighbors[i]);
+
+ std::vector<nemo::SM> sm_v;
+ s = g_pika_server->db()->ZRangebyscore(key, (double)min, (double)max, sm_v, false, false);
+ if (!s.ok()) {
+ res.SetRes(CmdRes::kErrOther, s.ToString());
+ return;
+ }
+ // Insert into result only if the point is within the search area.
+ for (int i = 0; i < sm_v.size(); ++i) {
+ double xy[2], real_distance;
+ GeoHashBits hash = { .bits = (uint64_t)sm_v[i].score, .step = GEO_STEP_MAX };
+ geohashDecodeToLongLatWGS84(hash, xy);
+ if(geohashGetDistanceIfInRadiusWGS84(longitude, latitude, xy[0], xy[1], distance, &real_distance)) {
+ NeighborPoint item;
+ item.member = sm_v[i].member;
+ item.score = sm_v[i].score;
+ item.distance = real_distance;
+ result.push_back(item);
+ }
+ }
+ }
+ // If using the count opiton
+ if (range.count) {
+ count_limit = range.count_limit;
+ } else {
+ count_limit = result.size();
+ }
+ // If using sort option
+ if (range.sort == Asc) {
+ std::sort(result.begin(), result.end(), sort_distance_asc);
+ } else if(range.sort == Desc) {
+ std::sort(result.begin(), result.end(), sort_distance_desc);
+ }
+ // For each the result
+ res.AppendArrayLen(count_limit);
+ for (int i = 0; i < count_limit; ++i) {
+ if (range.option_num != 0) {
+ res.AppendArrayLen(range.option_num+1);
+ }
+ // Member
+ res.AppendStringLen(result[i].member.size());
+ res.AppendContent(result[i].member);
+
+ // If using withdist option
+ if (range.withdist) {
+ double xy[2];
+ GeoHashBits hash = { .bits = (uint64_t)result[i].score, .step = GEO_STEP_MAX };
+ geohashDecodeToLongLatWGS84(hash, xy);
+ double distance = geohashGetDistance(longitude, latitude, xy[0], xy[1]);
+ distance = length_converter(distance, range.unit);
+ if (distance == -1) {
+ res.SetRes(CmdRes::kErrOther, "unsupported unit provided. please use m, km, ft, mi");
+ return;
+ }
+ char buf[32];
+ int64_t len = slash::d2string(buf, sizeof(buf), distance);
+ res.AppendStringLen(len);
+ res.AppendContent(buf);
+ }
+ // If using withcoord option
+ if (range.withcoord) {
+ res.AppendArrayLen(2);
+ double xy[2];
+ GeoHashBits hash = { .bits = (uint64_t)result[i].score, .step = GEO_STEP_MAX };
+ geohashDecodeToLongLatWGS84(hash, xy);
+
+ char longitude[32];
+ int64_t len = slash::d2string(longitude, sizeof(longitude), xy[0]);
+ res.AppendStringLen(len);
+ res.AppendContent(longitude);
+
+ char latitude[32];
+ len = slash::d2string(latitude, sizeof(latitude), xy[1]);
+ res.AppendStringLen(len);
+ res.AppendContent(latitude);
+ }
+ // If using withhash option
+ if (range.withhash) {
+ char buf[32];
+ int64_t len = slash::d2string(buf, sizeof(buf), result[i].score);
+ res.AppendStringLen(len);
+ res.AppendContent(buf);
+ }
+ }
+}
+
+void GeoRadiusCmd::DoInitial(PikaCmdArgsType &argv, const CmdInfo* const ptr_info) {
+ if (!ptr_info->CheckArg(argv.size())) {
+ res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoRadius);
+ return;
+ }
+ size_t argc = argv.size();
+ if (argc < 6) {
+ res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoRadius);
+ return;
+ }
+ key_ = argv[1];
+ slash::string2d(argv[2].data(), argv[2].size(), &range_.longitude);
+ slash::string2d(argv[3].data(), argv[3].size(), &range_.latitude);
+ slash::string2d(argv[4].data(), argv[4].size(), &range_.distance);
+ range_.unit = argv[5];
+ size_t pos = 6;
+ while (pos < argv.size()) {
+ if (!strcasecmp(argv[pos].c_str(), "withdist")) {
+ range_.withdist = true;
+ range_.option_num++;
+ } else if (!strcasecmp(argv[pos].c_str(), "withhash")) {
+ range_.withhash = true;
+ range_.option_num++;
+ } else if (!strcasecmp(argv[pos].c_str(), "withcoord")) {
+ range_.withcoord = true;
+ range_.option_num++;
+ } else if (!strcasecmp(argv[pos].c_str(), "count")) {
+ range_.count = true;
+ range_.count_limit = std::stoi(argv[++pos]);
+ } else if (!strcasecmp(argv[pos].c_str(), "asc")) {
+ range_.sort = Asc;
+ } else if (!strcasecmp(argv[pos].c_str(), "desc")) {
+ range_.sort = Desc;
+ } else {
+ res_.SetRes(CmdRes::kSyntaxErr);
+ return;
+ }
+ pos++;
+ }
+}
+
+void GeoRadiusCmd::Do() {
+ GetAllNeighbors(key_, range_, this->res_);
+}
+
+void GeoRadiusByMemberCmd::DoInitial(PikaCmdArgsType &argv, const CmdInfo* const ptr_info) {
+ if (!ptr_info->CheckArg(argv.size())) {
+ res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoRadius);
+ return;
+ }
+ size_t argc = argv.size();
+ if (argc < 5) {
+ res_.SetRes(CmdRes::kWrongNum, kCmdNameGeoRadius);
+ return;
+ }
+ key_ = argv[1];
+ range_.member = argv[2];
+ slash::string2d(argv[3].data(), argv[3].size(), &range_.distance);
+ range_.unit = argv[4];
+ size_t pos = 5;
+ while (pos < argv.size()) {
+ if (!strcasecmp(argv[pos].c_str(), "withdist")) {
+ range_.withdist = true;
+ range_.option_num++;
+ } else if (!strcasecmp(argv[pos].c_str(), "withhash")) {
+ range_.withhash = true;
+ range_.option_num++;
+ } else if (!strcasecmp(argv[pos].c_str(), "withcoord")) {
+ range_.withcoord = true;
+ range_.option_num++;
+ } else if (!strcasecmp(argv[pos].c_str(), "count")) {
+ range_.count = true;
+ range_.count_limit = std::stoi(argv[++pos]);
+ } else if (!strcasecmp(argv[pos].c_str(), "asc")) {
+ range_.sort = Asc;
+ } else if (!strcasecmp(argv[pos].c_str(), "desc")) {
+ range_.sort = Desc;
+ } else {
+ res_.SetRes(CmdRes::kSyntaxErr);
+ return;
+ }
+ pos++;
+ }
+}
+
+void GeoRadiusByMemberCmd::Do() {
+ nemo::Status s;
+ double score;
+ s = g_pika_server->db()->ZScore(key_, range_.member, &score);
+ if (s.ok()) {
+ double xy[2];
+ GeoHashBits hash = { .bits = (uint64_t)score, .step = GEO_STEP_MAX };
+ geohashDecodeToLongLatWGS84(hash, xy);
+ range_.longitude = xy[0];
+ range_.latitude = xy[1];
+ }
+ GetAllNeighbors(key_, range_, this->res_);
+}
\ No newline at end of file

diff:
@@ -60,7 +60,7 @@ static inline double rad_deg(double ang) { return ang / D_R; }
/* This function is used in order to estimate the step (bits precision)
* of the 9 search area boxes during radius queries. */
uint8_t geohashEstimateStepsByRadius(double range_meters, double lat) {
- if (range_meters == 0) return 26;
+ if (range_meters == 0) return 18;
int step = 1;
while (range_meters < MERCATOR_MAX) {
range_meters *= 2;

original_message: Support GEORADIUS AND GEORADIUSBYMEMBER


message: modify flags

diff:
@@ -199,7 +199,7 @@ enum CmdFlags {
kCmdFlagsNoPrior = 0, //default noprior
kCmdFlagsPrior = 128,
kCmdFlagsNoAdminRequire = 0, //default no need admin
- kCmdFlagsAdminRequire = 255
+ kCmdFlagsAdminRequire = 256
};

diff:
@@ -395,7 +395,7 @@ void InitCmdInfoTable() {
CmdInfo* geoposptr = new CmdInfo(kCmdNameGeoPos, -3, kCmdFlagsRead | kCmdFlagsGeo);
cmd_infos.insert(std::pair<std::string, CmdInfo*>(kCmdNameGeoPos, geoposptr));
////GeoDist
- CmdInfo* geodistptr = new CmdInfo(kCmdNameGeoDist, -3, kCmdFlagsWrite | kCmdFlagsGeo);
+ CmdInfo* geodistptr = new CmdInfo(kCmdNameGeoDist, -3, kCmdFlagsRead | kCmdFlagsGeo);
cmd_infos.insert(std::pair<std::string, CmdInfo*>(kCmdNameGeoDist, geodistptr));
////GeoHash
CmdInfo* geohashptr = new CmdInfo(kCmdNameGeoHash, -3, kCmdFlagsRead | kCmdFlagsGeo);

original_message: modify flags