message: fix some bugs of ghttp and modify gbackup and grestore

diff:
using namespace std;
#define SYSTEM_PATH "data/system/system.nt"
-#define DEFALUT_BACKUP_PATH "./backups"
+#define DEFALUT_BACKUP_PATH "backups"
int copy(string src_path, string dest_path)
{
+ string sys_cmd;
if(!boost::filesystem::exists(src_path)){
//check the source path
cout << "Source Path Error, Please Check It Again!" << endl;
@@ -28,11 +29,11 @@ int copy(string src_path, string dest_path)
}
if(!boost::filesystem::exists(dest_path)){
//check the destnation path
- cout << "Destnation Path Error, Please Check It Again!" << endl;
- return -1;
+ sys_cmd = "mkdir -p ./" + dest_path;
+ system(sys_cmd.c_str());
}
- string sys_cmd;
- sys_cmd = "cp -r " + src_path + ' ' + dest_path ;
+
+ sys_cmd = "cp -r " + src_path + " ./" + dest_path ;
system(sys_cmd.c_str());
return 0;// success
}
@@ -41,13 +42,12 @@ int
main(int argc, char * argv[])
{
Util util;
- string db_name, db_path, backup_path, build_path;
+ string db_name, backup_path;
if(argc < 2)
{
//output help info here
cout << "the usage of gbackup: " << endl;
- cout << "./bin/gbackup your_database_name your_backup_path(optional) your_database_path(optional)" << endl;
- cout << "defalut database_path = ./ " << endl;
+ cout << "./bin/gbackup your_database_name your_backup_path(optional) "<< endl;
cout << "defalut backup_path = ./backups" << endl;
cout << "the path should not include your database's name!" << endl;
return 0;
@@ -55,13 +55,11 @@ main(int argc, char * argv[])
db_name = string(argv[1]);
if(argc > 2)
backup_path = string(argv[2]);
- if(argc > 3)
- build_path = string(argv[3]);
+
cout << "gbackup..." << endl;
{
cout << "argc: " << argc << "\t";
cout << "DB_name:" << db_name << "\t";
- cout << "Build_path: " << db_path << "\t";
cout << "Backup_path: " << backup_path << "\t";
cout << endl;
}
@@ -85,17 +83,30 @@ main(int argc, char * argv[])
//query database_name build_path
//insert database_name backup_path
+ if(backup_path == "") backup_path = DEFALUT_BACKUP_PATH;
+ if(backup_path == "." || backup_path == "./" ){
+ cout << "Backup Path Can not be root, Backup Failed!" << endl;
+ return 0;
+ }
+ if(backup_path[0] == '/') backup_path = '.' + backup_path;
+ if(backup_path[backup_path.length() - 1] == '/') backup_path = backup_path.substr(0, backup_path.length() - 1);
+
+ backup_path = "./" + backup_path;
+ string db_path = db_name + ".db";
+
//copy
- if(db_path == "")
- db_path = db_name + ".db";
if(backup_path == "") backup_path = DEFALUT_BACKUP_PATH;
int ret = copy(db_path, backup_path);
if(ret == 1){
- cout << "Database Path Error, Backup Failed!" << endl;
- }else if(ret == -1){
- cout << "Backup Path Error, Backup Failed!" << endl;
- }else{
+ cout << "Database Name Error, Backup Failed!" << endl;
+ }
+ else{
string time = Util::get_date_time();
+ string timestamp = Util::get_timestamp();
+ backup_path = backup_path + "/" + db_path;
+ string _backup_path = backup_path + "_" + timestamp;
+ string sys_cmd = "mv " + backup_path + " " + _backup_path;
+ system(sys_cmd.c_str());
cout << "Time:" + time << endl;
cout << "DB:" + db_name + " Backup done!" << endl;
}

diff:
@@ -108,6 +108,8 @@ bool backup_handler(const HttpServer& server, const shared_ptr<HttpServer::Respo
bool restore_handler(const HttpServer& server, const shared_ptr<HttpServer::Response>& response, const shared_ptr<HttpServer::Request>& request, string RequestType);
+bool auto_backup_handler(const HttpServer& server, const shared_ptr<HttpServer::Response>& response, const shared_ptr<HttpServer::Request>& request, string RequestType);
+
bool getCoreVersion_handler(const HttpServer& server, const shared_ptr<HttpServer::Response>& response, const shared_ptr<HttpServer::Request>& request, string RequestType);
bool getAPIVersion_handler(const HttpServer& server, const shared_ptr<HttpServer::Response>& response, const shared_ptr<HttpServer::Request>& request, string RequestType);
@@ -3966,7 +3968,7 @@ void user_thread(const shared_ptr<HttpServer::Response>& response, const shared_
pthread_rwlock_unlock(&users_map_lock);
}
- else if(type == "add_query" || type == "add_load" || type == "add_unload" || type == "add_update" || type == "add_backup" || type == "add_restore" || "add_export")
+ else if(type == "add_query" || type == "add_load" || type == "add_unload" || type == "add_update" || type == "add_backup" || type == "add_restore" || type == "add_export")
{
if(username2 == ROOT_USERNAME)
{
@@ -4448,6 +4450,11 @@ void showUser_thread(const shared_ptr<HttpServer::Response>& response, const sha
_restore_db.SetString(restore_db.c_str(), restore_db.length(), allocator);
obj.AddMember("restore privilege", _restore_db, allocator);
+ string export_db = it->second->getexport();
+ Value _export_db;
+ _export_db.SetString(export_db.c_str(), export_db.length(), allocator);
+ obj.AddMember("export privilege", _export_db, allocator);
+
json_array.PushBack(obj, allocator);
}
resDoc.AddMember("ResponseBody", json_array, allocator);
@@ -4805,6 +4812,9 @@ void DB2Map()
}else if(type == "has_backup_priv")
{
user->backup_priv.insert(db_name);
+ }else if(type == "has_export_priv")
+ {
+ user->export_priv.insert(db_name);
}
}
//users.insert(pair<std::string, struct User*>(username, &user));
@@ -5302,7 +5312,9 @@ void restore_thread(const shared_ptr<HttpServer::Response>& response, const shar
return;
}
- //check if database named [db_name] is already build.
+ if(path[0] == '/') path = '.' + path;
+ if(path[path.length() - 1] == '/') path = path.substr(0, path.length()-1);
+
if(!boost::filesystem::exists(path)){
string error = "Backup Path Error, Restore Failed";
string resJson = CreateJson(307, error, 0);
@@ -5325,7 +5337,7 @@ void restore_thread(const shared_ptr<HttpServer::Response>& response, const shar
pthread_rwlock_unlock(&already_build_map_lock);
return;
}
- if(addPrivilege(username, "query", db_name) == 0 || addPrivilege(username, "load", db_name) == 0 || addPrivilege(username, "unload", db_name) == 0 || addPrivilege(username, "backup", db_name) == 0 || addPrivilege(username, "restore", db_name) == 0)
+ if(addPrivilege(username, "query", db_name) == 0 || addPrivilege(username, "load", db_name) == 0 || addPrivilege(username, "unload", db_name) == 0 || addPrivilege(username, "backup", db_name) == 0 || addPrivilege(username, "restore", db_name) == 0 || addPrivilege(username, "export", db_name) == 0)
{
string error = "add query or load or unload or backup or restore privilege failed.";
string resJson = CreateJson(912, error, 0);
@@ -5359,6 +5371,8 @@ void restore_thread(const shared_ptr<HttpServer::Response>& response, const shar
return;
}
//restore
+ string sys_cmd = "rm -rf " + db_name + ".db";
+ system(sys_cmd.c_str());
pthread_rwlock_wrlock(&databases_map_lock);
int ret = copy(path, DB_PATH);
@@ -5387,10 +5401,9 @@ void restore_thread(const shared_ptr<HttpServer::Response>& response, const shar
pthread_rwlock_unlock(&(it_already_build->second->db_lock));
path = Util::get_folder_name(path, db_name);
- string sys_cmd = "mv " + path + " " + db_name + ".db";
+ sys_cmd = "mv " + path + " " + db_name + ".db";
system(sys_cmd.c_str());
-
cout << "database restore done." << endl;
string success = "Database restore successfully.";
string resJson = CreateJson(0, success, 0);
@@ -5405,3 +5418,9 @@ bool restore_handler(const HttpServer& server, const shared_ptr<HttpServer::Resp
t.detach();
return true;
}
+
+
+bool auto_backup_handler(const HttpServer& server, const shared_ptr<HttpServer::Response>& response, const shared_ptr<HttpServer::Request>& request, string RequestType)
+{
+
+}
\ No newline at end of file

diff:
@@ -38,30 +38,25 @@ int
main(int argc, char * argv[])
{
Util util;
- string db_name, backup_path, db_path;
+ string db_name, backup_path, db_path, path;
if(argc < 2)
{
//output help info here
cout << "the usage of grestore: " << endl;
- cout << "./bin/restore your_database_name your_backup_path(optional) your_database_path(optional)" << endl;
- cout << "defalut backup_path = ./backups" << endl;
- cout << "defalut database_path = ./ " << endl;
- cout << "the path should not include your database's name!" << endl;
+ cout << "./bin/restore your_database_name your_backup_path" << endl;
+ cout << "the path should include your database folder name!" << endl;
return 0;
}
db_name = string(argv[1]);
if(argc > 2)
backup_path = string(argv[2]);
- if(argc > 3)
- db_path = string(argv[3]);
cout << "gbackup..." << endl;
{
cout << "argc: " << argc << "\t";
cout << "DB_name:" << db_name << "\t";
- cout << "Build_path: " << db_path << "\t";
cout << "Backup_path: " << backup_path << "\t";
cout << endl;
}
@@ -80,35 +75,65 @@ main(int argc, char * argv[])
return -1;
}
-//TODO1: We need a column in system.db :
-//last restore time;
-//TODO2:
-//We can restore a deleted database
-//if we insert the new database's info into the system.db
-//then we restore the files and we rebuild it
+ if(backup_path[0] == '/') backup_path = '.' + backup_path;
+ if(backup_path[backup_path.length() - 1] == '/') backup_path = backup_path.substr(0, backup_path.length()-1);
- //copy
+ if(!boost::filesystem::exists(backup_path)){
+ cout << "Backup Path Error, Restore Failed" << endl;
+ return 0;
+ }
- if(backup_path == "")
- backup_path = DEFALUT_BACKUP_PATH;
- backup_path = backup_path + "/" + db_name + ".db";
- if(db_path == "") db_path = DEFALUT_BUILD_PATH;
+ //system.db
+ Database system_db("system");
+ system_db.load();
- //delete
- string _db_path = db_path + "/" + db_name + ".db";
- string sys_cmd = "rm -rf " + _db_path;
- system(sys_cmd.c_str());
+ string sparql = "ASK WHERE{<" + db_name + "> <database_status> \"already_built\".}";
+ ResultSet ask_rs;
+ FILE* ask_ofp = stdout;
+ system_db.query(sparql, ask_rs, ask_ofp);
+ if (ask_rs.answer[0][0] == "false")
+ {
+ cout << "The database does not exist. Rebuild" << endl;
+ string time = Util::get_backup_time(backup_path, db_name);
+ if(time.size() == 0){
+ cout << "Backup Path Does not Match DataBase Name, Restore Failed" << endl;
+ return 0;
+ }
+ string sparql = "INSERT DATA {<" + db_name + "> <database_status> \"already_built\"." + "<" + db_name + "> <built_by> <root>."
+ + "<" + db_name + "> <built_time> \"" + time + "\".}";
+ ResultSet _rs;
+ FILE* ofp = stdout;
+ string msg;
+ int ret = system_db.query(sparql, _rs, ofp);
+
+ if (ret >= 0)
+ msg = "update num : " + Util::int2string(ret);
+ else {
+ //update error
+ cout << "Rebuild Error, Restore Failed" << endl;
+ return 0;
+ }
+ cout << msg << endl;
+ }
+
+ int ret = copy(backup_path, DEFALUT_BUILD_PATH);
- int ret = copy(backup_path, db_path);
if(ret == 1){
cout << "Backup Path Error, Restore Failed!" << endl;
- }else if(ret == -1){
- cout << "Database Path Error, Restore Failed!" << endl;
}else{
//TODO update the in system.db
string time = Util::get_date_time();
cout << "Time:" + time << endl;
cout << "DB:" + db_name + " Restore done!" << endl;
}
+
+ db_path = db_name + ".db";
+ string sys_cmd = "rm -rf " + db_path;
+ system(sys_cmd.c_str());
+
+ path = Util::get_folder_name(backup_path, db_name);
+ sys_cmd = "mv " + path + ' ' + db_path;
+ system(sys_cmd.c_str());
+
return 0;
}
\ No newline at end of file


message: fix a bug of initsys()

change_type: MODIFY

diff:
@@ -72,7 +72,7 @@ string querySys(string sparql);
bool sysrefresh();
bool updateSys(string sparql);
bool initSys();
-bool refreshSys();
+//bool refreshSys();
int db_reload(string db_name);
//bool doQuery(string format, string db_query, const HttpServer& server, const shared_ptr<HttpServer::Response>& response, const shared_ptr<HttpServer::Request>& request);
@@ -5626,7 +5626,6 @@ bool initSys()
system_database = new Database("system");
bool flag = system_database->build(SYSTEM_PATH);
- cout << "rebuild success" << endl;
if(flag)
{
cout << "import RDF file to database done." << endl;
@@ -5646,11 +5645,12 @@ bool initSys()
delete system_database;
system_database = new Database("system");
system_database->load();
- cout << "system_database load" << endl;
+ cout << "system_database loaded" << endl;
Util::init_backuplog();
return true;
}
+/*
bool refreshSys()
{
delete system_database;
@@ -5659,6 +5659,7 @@ bool refreshSys()
int flag = system_database->load();
return flag;
}
+*/
int copy(string src_path, string dest_path)
{
@@ -6126,14 +6127,13 @@ void init_thread(const shared_ptr<HttpServer::Response>& response, const shared_
users.clear();
//rebuild system.db
pthread_rwlock_unlock(&users_map_lock);
-
vector<string> db_names;
Util::split(db_list, " ", db_names);
-
- pthread_rwlock_wrlock(&(databases_map_lock));
+ db_names.push_back("system");
+ pthread_rwlock_wrlock(&databases_map_lock);
pthread_rwlock_wrlock(&(already_build_map_lock));
- std::map<std::string, struct DBInfo *>::iterator it_already_build = already_build.begin();
+
//check the db_names
vector<string>::iterator it;
for(it = db_names.begin(); it != db_names.end(); it++){
@@ -6144,12 +6144,19 @@ void init_thread(const shared_ptr<HttpServer::Response>& response, const shared_
if(it == db_names.end()) break;
}
}
+ //unload all databases
+ for(std::map<std::string, Database *>::iterator iter = databases.begin(); iter != databases.end(); iter++){
+ Database *current_database = iter->second;
+ delete current_database;
+ current_database = NULL;
+ databases.erase(iter->first);
+ }
+ std::map<std::string, struct DBInfo *>::iterator it_already_build = already_build.begin();
//delete files
for(it_already_build = already_build.begin(); it_already_build != already_build.end(); it_already_build++)
{
string db_name = it_already_build->first;
- if(db_name == "system") continue;
if(find(db_names.begin(), db_names.end(), db_name) == db_names.end())
{
string cmd;
@@ -6160,20 +6167,19 @@ void init_thread(const shared_ptr<HttpServer::Response>& response, const shared_
}
system(cmd.c_str());
}
+ already_build.erase(db_name);
}
- cout << "check success" << endl;
- //clear already build
- already_build.clear();
- //clear loaded databases
- databases.clear();
+ //clear already_build
+ cout << "clear success" << endl;
int flag = initSys();
struct DBInfo *temp_db = new DBInfo("system");
temp_db->setCreator("root");
already_build.insert(pair<std::string, struct DBInfo *>("system", temp_db));
databases.insert(pair<std::string, Database *>("system", system_database));
-
+ pthread_rwlock_unlock(&databases_map_lock);
+ pthread_rwlock_unlock(&(already_build_map_lock));
if(!flag){
cout << "system.db rebuild failed " << endl;
return;
@@ -6195,12 +6201,11 @@ void init_thread(const shared_ptr<HttpServer::Response>& response, const shared_
sparql += "}";
updateSys(sparql);
- //reload system.db
- refreshSys();
//rebuild datastructure
DB2Map();
- pthread_rwlock_unlock(&already_build_map_lock);
- pthread_rwlock_unlock(&(databases_map_lock));
+ sysrefresh();
+ cout << "DB2Map success " << endl;
+
string success = "system.db initialize success.";
string resJson = CreateJson(918, success, 0);
*response << "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " << resJson.length() << "\r\n\r\n" << resJson;

license: BSD 3-Clause New or Revised License

repo: pkumod/gstore

original_message: fix a bug of initsys()


message: fix bugs of initsys

diff:
@@ -5646,7 +5646,11 @@ bool initSys()
system_database = new Database("system");
system_database->load();
cout << "system_database loaded" << endl;
- Util::init_backuplog();
+ string cmd;
+ cmd = "mv ./port.txt ./system.db/";
+ system(cmd.c_str());
+ cmd = "mv ./password" + Util::int2string(port) + ".txt ./system.db/";
+ system(cmd.c_str());
return true;
}
@@ -6124,16 +6128,16 @@ void init_thread(const shared_ptr<HttpServer::Response>& response, const shared_
string root_password = (users.find(ROOT_USERNAME))->second->getPassword();
if(username == ROOT_USERNAME && password == root_password){
+ cout << "init begin....." << endl;
users.clear();
//rebuild system.db
- pthread_rwlock_unlock(&users_map_lock);
vector<string> db_names;
- Util::split(db_list, " ", db_names);
db_names.push_back("system");
+ Util::split(db_list, " ", db_names);
pthread_rwlock_wrlock(&databases_map_lock);
pthread_rwlock_wrlock(&(already_build_map_lock));
-
+ cout << "check the db_names" << endl;
//check the db_names
vector<string>::iterator it;
for(it = db_names.begin(); it != db_names.end(); it++){
@@ -6144,22 +6148,37 @@ void init_thread(const shared_ptr<HttpServer::Response>& response, const shared_
if(it == db_names.end()) break;
}
}
+ cout << "unload all databases" << endl;
+ cout << "loaded databases num: " << databases.size() << endl;
//unload all databases
for(std::map<std::string, Database *>::iterator iter = databases.begin(); iter != databases.end(); iter++){
+ if(iter->first == "system") continue;
Database *current_database = iter->second;
delete current_database;
current_database = NULL;
databases.erase(iter->first);
}
+ cout << "unload all databases done" << endl;
std::map<std::string, struct DBInfo *>::iterator it_already_build = already_build.begin();
+ //backup
+ cout << "backup server_files" << endl;
+ string cmd;
+ cmd = "mv ./system.db/port.txt ./";
+ system(cmd.c_str());
+ cmd = "mv ./system.db/password" + Util::int2string(port) + ".txt ./";
+ system(cmd.c_str());
//delete files
for(it_already_build = already_build.begin(); it_already_build != already_build.end(); it_already_build++)
{
string db_name = it_already_build->first;
- if(find(db_names.begin(), db_names.end(), db_name) == db_names.end())
+ //if(db_name == "system") cout << "system system system" << endl;
+ if(find(db_names.begin(), db_names.end(), db_name) == db_names.end() || db_name == "system")
{
- string cmd;
+ if(db_name == "system"){
+ delete system_database;
+ system_database = NULL;
+ }
if(is_backup == "true"){
cmd = "mv " + db_name + ".db " + db_name + ".bak";
}else{
@@ -6192,7 +6211,7 @@ void init_thread(const shared_ptr<HttpServer::Response>& response, const shared_
for(int i = 0; i < db_names.size(); i++)
{
string db_name = db_names[i];
- if(db_name == "") continue;
+ if(db_name == "" || db_name == "system") continue;
Util::add_backuplog(db_name);
sparql = sparql + "<" + db_name + "> <database_status> \"already_built\".";
sparql = sparql + "<" + db_name + "> <built_by> <root>.";
@@ -6201,9 +6220,10 @@ void init_thread(const shared_ptr<HttpServer::Response>& response, const shared_
sparql += "}";
updateSys(sparql);
- //rebuild datastructure
- DB2Map();
+ ////rebuild datastructure
sysrefresh();
+ DB2Map();
+ pthread_rwlock_unlock(&users_map_lock);
cout << "DB2Map success " << endl;
string success = "system.db initialize success.";

diff:
@@ -150,11 +150,54 @@ int main(int argc, char * argv[])
return 0;
}
}
-
- //build system.db
Util util;
+ ResultSet _rs;
+ {
+ Database system_db("system");
+ system_db.load();
+ string sparql = "select ?s where{?s <database_status> \"already_built\".}";
+ FILE* ofp = stdout;
+ int ret = system_db.query(sparql, _rs, ofp);
+ }
+ if(argc > 2){
+ cout << "db reserved db reserved db reserved db reserved db reserved " << endl;
+ vector<string> db_names;
+ for(int i = 2; i < argc; i++){
+ cout << argv[i] << endl;
+ db_names.push_back(argv[i]);
+ }
+ for (int i = 0; i < _rs.ansNum; i++)
+ {
+ string db_name = _rs.answer[i][0];
+ db_name.erase(0,1);
+ db_name.pop_back();
+ //cout << "db num:db num:db num:db num:db num:db num:db num:db num:" << db_name << endl;
+ if(find(db_names.begin(), db_names.end(), db_name) == db_names.end() || db_name == "system")
+ {
+ string cmd;
+ cmd = "rm -r " + db_name + ".db";
+ system(cmd.c_str());
+ }
+ }
+ }
+ else{
+ string cmd;
+ cmd = "rm -r system.db";
+ system(cmd.c_str());
+ cout << _rs.ansNum << endl;
+ for (int i = 0; i < _rs.ansNum; i++)
+ {
+ string db_name = _rs.answer[i][0];
+ db_name.erase(0,1);
+ db_name.pop_back();
+ string cmd;
+ cmd = "rm -r " + db_name + ".db";
+ system(cmd.c_str());
+ }
+ }
+ //build system.db
string _db_path = "system";
- string _rdf = "data/system/system.nt";
+ string _rdf = "./data/system/system.nt";
Database* _db = new Database(_db_path);
bool flag = _db->build(_rdf);
if (flag)
@@ -197,7 +240,6 @@ int main(int argc, char * argv[])
}
}
sparql = sparql + "}";
- ResultSet _rs;
FILE* ofp = stdout;
string msg;
int ret = _db->query(sparql, _rs, ofp);

original_message: fix bugs of initsys


message: fix  a bug of ginit

diff:
@@ -152,6 +152,8 @@ int main(int argc, char * argv[])
}
Util util;
ResultSet _rs;
+ bool no_sysdb = false;
+ if(boost::filesystem::exists("system.db"))
{
Database system_db("system");
system_db.load();
@@ -159,6 +161,8 @@ int main(int argc, char * argv[])
FILE* ofp = stdout;
int ret = system_db.query(sparql, _rs, ofp);
}
+ else
+ no_sysdb = true;
if(argc > 2){
cout << "db reserved db reserved db reserved db reserved db reserved " << endl;
vector<string> db_names;

original_message: fix  a bug of ginit


message: add transaction cpp api and usage example

old_path: api/http/cpp/example/Makefile

new_path: api/http/cpp/example/Makefile

diff:
CC=g++ -std=c++11
#CC=ccache g++
-all: POST-example GET-example Benchmark
+all: POST-example GET-example Benchmark Transaction-example
Benchmark: Benchmark.o
- $(CC) -o Benchmark Benchmark.o -L../lib -lclient -lcurl -lpthread
+ $(CC) -o Benchmark Benchmark.o -L../lib -lclient -lpthread -lcurl
GET-example: GET-example.o
- $(CC) -o GET-example GET-example.o -I../ -lcurl -L../lib -lclient
+ $(CC) -o GET-example GET-example.o -I../ -L../lib -lclient -lcurl
POST-example: POST-example.o
- $(CC) -o POST-example POST-example.o -I../ -lcurl -L../lib -lclient
+ $(CC) -o POST-example POST-example.o -I../ -L../lib -lclient -lcurl
+
+Transaction-example:Transaction-example.o
+ $(CC) -o Transaction-example Transaction-example.o -I../ -L../lib -lclient -lcurl
GET-example.o: GET-example.cpp ../client.cpp ../client.h
$(CC) -c -I../ GET-example.cpp -o GET-example.o
@@ -19,5 +22,8 @@ Benchmark.o: Benchmark.cpp ../client.cpp ../client.h
POST-example.o: POST-example.cpp ../client.cpp ../client.h
$(CC) -c -I../ POST-example.cpp -o POST-example.o
+Transaction-example.o: Transaction-example.cpp ../client.cpp ../client.h
+ $(CC) -c -I../ Transaction-example.cpp -o Transaction-example.o
+
clean:
- rm -rf *.o POST-example GET-example Benchmark
+ rm -rf *.o POST-example GET-example Benchmark Transaction-example


message: fix bugs in transaction

change_type: ADD

diff:
+#include <iostream>
+#include <set>
+#include <vector>
+#include <deque>
+#include <algorithm>
+#include <fstream>
+#include <mutex>
+#include <thread>
+#include "../Database/Txn_manager.h"
+
+using namespace std;
+
+struct task
+{
+ string op1;
+ string op2;
+ string op3;
+ task() {op1 = op2 = op3 = "";}
+};
+
+class task_manager
+{
+private:
+ ifstream insert_if;
+ ifstream delete_if;
+ mutex l;
+ const string insert_filename = "./scripts/insert.nt";
+ const string delete_filename = "./scripts/delete.nt";
+public:
+ task get_task();
+ task_manager();
+ ~task_manager();
+};
+
+task_manager::task_manager()
+{
+ insert_if.open(insert_filename, ios::in);
+ delete_if.open(delete_filename, ios::in);
+}
+
+task_manager::~task_manager()
+{
+ insert_if.close();
+ delete_if.close();
+
+}
+
+task task_manager::get_task()
+{
+ task t;
+ l.lock();
+ string sparql;
+ getline(insert_if, sparql);
+ if(insert_if.eof()) {
+ l.unlock();
+ return t;
+ }
+ t.op1 = sparql;
+ getline(insert_if, sparql);
+ t.op2 = sparql;
+ getline(delete_if, sparql);
+ t.op3 = sparql;
+ l.unlock();
+ return t;
+}
+
+class writer
+{
+private:
+ ofstream of;
+ mutex lock;
+ atomic<int> n;
+ const string out_filename = "./scripts/statics.txt";
+public:
+ writer();
+ ~writer();
+ void write_task(task &t, int code);
+ void write_num(int num);
+ void write_dataset(vector<IDSet>& readset, vector<IDSet>& writeset, txn_id_t TID);
+};
+
+writer::writer()
+{
+ of.open(out_filename, ios::out);
+ n.store(1);
+}
+
+writer::~writer()
+{
+ of.close();
+}
+
+void writer::write_task(task &t, int code)
+{
+ lock.lock();
+ of << "#" << code << endl << t.op1 << endl << t.op2 << endl << t.op3 << endl;
+ n++;
+ lock.unlock();
+}
+
+void writer::write_num(int num)
+{
+ lock.lock();
+ of << "###" << n.load() << ": " << num << endl;
+ n++;
+ lock.unlock();
+}
+
+void writer::write_dataset(vector<IDSet>& readset, vector<IDSet>& writeset, txn_id_t TID)
+{
+ lock.lock();
+ of << TID << endl;
+ of << "read set: " << endl;
+ for(int i = 0; i < 3; i++){
+ for(auto id: readset[i])
+ {
+ of << id << " ";
+ }
+ of << endl;
+ }
+ of << "write set: " << endl;
+ for(int i = 0; i < 3; i++){
+ for(auto id: writeset[i])
+ {
+ of << id << " ";
+ }
+ of << endl;
+ }
+ of << endl;
+ lock.unlock();
+}
+
+task_manager t_m;
+writer w;
+
+void func()
+{
+ task t;
+ int cnt = 0;
+ while(1){
+ t = t_m.get_task();
+ if(t.op1 == "") break;
+ //start transaction
+ w.write_task(t, 1);
+ cnt++;
+ }
+ cout << this_thread::get_id() << " " << cnt << endl;
+}
+
+void run_transaction(int isolevel, Txn_manager& txn_m)
+{
+ task t;
+ while(true){
+ t = t_m.get_task();
+ if(t.op1 == "") return;
+ string res, sparql;
+ int cnt = 0;
+ int base_time = 1000;
+ txn_id_t TID;
+ int num = 0;
+ while(true){
+ TID = txn_m.Begin(static_cast<IsolationLevelType>(isolevel));
+ //cerr << "transaction #" << TID << endl;
+ int ret;
+ cnt++;
+ sparql = "insert data {" + t.op1 + "}";
+ ret = txn_m.Query(TID, sparql, res);
+ if(ret < 0) {
+ //cerr << "insert 1 failed ! " << ret << endl;
+ usleep(base_time);
+ base_time = min(base_time*2, 1000000);
+ continue;
+ }
+ sparql = "insert data {" + t.op2 + "}";
+ ret = txn_m.Query(TID, sparql, res);
+ if(ret < 0) {
+ //cerr << "insert 2 failed ! " << ret << endl;
+ usleep(base_time);
+ base_time = min(base_time*2, 1000000);
+ continue;
+ }
+ sparql = "delete data {" + t.op3 + "}";
+ ret = txn_m.Query(TID, sparql, res);
+ if(ret < 0) {
+ //cerr << "delete 1 failed ! " << ret << endl;
+ usleep(base_time);
+ base_time = min(base_time*2, 1000000);
+ continue;
+ }
+ if(txn_m.Commit(TID) == 0) break;
+ }
+ w.write_num(cnt);
+ }
+}
+
+void create_versions(Txn_manager& txn_m)
+{
+ string query = "select ?v {<V1> <R1> ?v.}";
+ string insert1 = "insert data { <V1> <R1> \"5\"^^<http://www.w3.org/2001/XMLSchema#integer> }";
+ string insert2 = "insert data { <V1> <R1> \"15\"^^<http://www.w3.org/2001/XMLSchema#integer> }";
+ string delete1 = "delete data { <V1> <R1> \"5\"^^<http://www.w3.org/2001/XMLSchema#integer> }";
+ string delete2 = "delete data { <V1> <R1> \"15\"^^<http://www.w3.org/2001/XMLSchema#integer> }";
+
+ string res;
+
+ txn_id_t id = txn_m.Begin();
+ txn_m.Query(id, query, res);
+ txn_m.print_txn_dataset(id);
+ txn_m.Commit(id);
+
+ id = txn_m.Begin();
+ txn_m.Query(id, insert1, res);
+ txn_m.print_txn_dataset(id);
+ txn_m.Commit(id);
+
+ id = txn_m.Begin();
+ txn_m.Query(id, insert2, res);
+ txn_m.print_txn_dataset(id);
+ txn_m.Commit(id);
+
+ id = txn_m.Begin();
+ txn_m.Query(id, delete1, res);
+ txn_m.print_txn_dataset(id);
+ txn_m.Commit(id);
+
+ id = txn_m.Begin();
+ txn_m.Query(id, delete2, res);
+ txn_m.print_txn_dataset(id);
+ txn_m.Commit(id);
+}
+
+void no_txn_update(Database &db)
+{
+ const string insert_filename = "./scripts/insert.nt";
+ const string delete_filename = "./scripts/delete.nt";
+ fstream in;
+
+ string line, sparql, res;
+ ResultSet rs;
+ in.open(insert_filename, ios::in);
+ while(getline(in, line))
+ {
+ sparql = "insert data {" + line + "}";
+ FILE *fp = stdout;
+ db.query(sparql, rs, fp);
+ }
+ in.close();
+
+ in.open(delete_filename, ios::in);
+ while(getline(in, line))
+ {
+ sparql = "delete data {" + line + "}";
+ FILE *fp = stdout;
+ db.query(sparql, rs, fp);
+ }
+ in.close();
+}
+
+bool single_txn(int threads_num, Txn_manager& txn_m)
+{
+ txn_id_t TID = txn_m.Begin(static_cast<IsolationLevelType>(3));
+ ifstream in;
+ string line, sparql, res;
+ in.open("./scripts/insert.nt", ios::in);
+ int num = 0;
+ while(getline(in, line))
+ {
+ sparql = "insert data {" + line + "}";
+ int ret = txn_m.Query(TID, sparql, res);
+ num += ret;
+ //txn_m.Get_Transaction(TID)->print_all();
+ //if(ret != 0) cerr << "wrong answer!" << endl;
+ // getline(in, line);
+ // sparql = "insert data {" + line + "}";
+ // ret = txn_m.Query(TID, sparql, res);
+ //txn_m.Get_Transaction(TID)->print_all();
+ //if(ret != 0) cerr << "wrong answer!" << endl;
+ }
+ in.close();
+ in.open("./scripts/delete.nt", ios::in);
+ while(getline(in, line))
+ {
+ sparql = "delete data{" + line + "}";
+ int ret = txn_m.Query(TID, sparql, res);
+ //if(ret == 0) cerr << "wrong answer!wrong answer!wrong answer!wrong answer!" << endl;
+ num += ret;
+ }
+ //txn_m.Get_Transaction(TID)->print_all();
+ in.close();
+ txn_m.Commit(TID);
+ cout << "update num: " << num << endl;
+ return true;
+}
+
+void check_results(int threads_num, Txn_manager& txn_m)
+{
+ txn_id_t TID = txn_m.Begin();
+ ifstream in;
+ in.open("./scripts/insert.nt", ios::in);
+ for(int i = 0; i < threads_num; i++)
+ {
+ string line, sparql, res;
+ getline(in, line);
+ sparql = "ask where {" + line + "}";
+ int ret = txn_m.Query(TID, sparql, res);
+ cout << res << endl;
+ getline(in, line);
+ sparql = "ask where {" + line + "}";
+ ret = txn_m.Query(TID, sparql, res);
+ cout << res << endl;
+ }
+ in.close();
+ in.open("./scripts/delete.nt", ios::in);
+ for(int i = 0; i < threads_num; i++)
+ {
+ string line, sparql, query, res;
+ getline(in, line);
+ sparql = "ask where{" + line + "}";
+ int ret = txn_m.Query(TID, sparql, res);
+ cout << res << endl;
+ }
+ txn_m.Commit(TID);
+}
+int main(int argc, char* argv[])
+{
+ Util util;
+
+ string db_folder = "lubm";
+
+ Database _db(db_folder);
+ _db.load();
+ cout << "finish loading" << endl;
+ Txn_manager txn_m(&_db, string("lubm"));
+ int threads_num;
+ //threads_num = thread::hardware_concurrency()-1;
+ threads_num = 17;
+ vector<thread> pool(threads_num);
+ int n = pool.size();
+ cout << n;
+ for(int i = 0; i < n; i++)
+ {
+ pool[i] = thread(run_transaction, 1 , ref(txn_m));
+ //pool[i] = thread(create_versions, ref(txn_m));
+ //pool[i] = thread(func);
+ }
+
+ for(int i = 0; i < n; i++)
+ {
+ pool[i].join();
+ }
+
+ // single_txn(61, txn_m);
+ // check_results(61, txn_m);
+ // no_txn_update(_db);
+ // single_txn(61, txn_m);
+ cout << "workload finished!" << endl;
+ return 0;
+}
\ No newline at end of file

original_message: fix bugs in transaction


message: fix the compile bugs raised by merge

diff:
@@ -102,7 +102,7 @@ public:
//MVCC
void transaction_rollback(shared_ptr<Transaction> txn);
void transaction_commit(shared_ptr<Transaction> txn);
- void version_clean();
+ void version_clean(vector<unsigned> &sub_ids ,vector<unsigned>& obj_ids, vector<unsigned>& obj_literal_ids, vector<unsigned> &pre_ids);
private:
string name;
string store_path;
@@ -139,6 +139,7 @@ private:
StringIndex* stringindex;
Join* join;
+ enum class UPDATE_TYPE{ SUBJECT_INSERT, SUBJECT_REMOVE, PREDICATE_INSERT, PREDICATE_REMOVE, OBJECT_INSERT, OBJECT_REMOVE};
//metadata of this database: sub_num, pre_num, obj_num, literal_num, etc.
string db_info_file;
@@ -294,6 +295,13 @@ private:
//bool insert(const vector<TripleWithObjType>& _triples, vector<int>& _vertices, vector<int>& _predicates);
unsigned remove(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _triple_num, bool _is_restore=false, shared_ptr<Transaction> txn = nullptr);
+
+ unsigned batch_insert(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _triple_num, bool _is_restore=false , shared_ptr<Transaction> txn = nullptr);
+ unsigned batch_remove(const TripleWithObjType* _triples, TYPE_TRIPLE_NUM _triple_num, bool _is_restore=false , shared_ptr<Transaction> txn = nullptr);
+
+ void sub_batch_update(vector<ID_TUPLE> id_tuples, TYPE_TRIPLE_NUM _triple_num, unsigned &update_num, UPDATE_TYPE type, shared_ptr<Transaction> txn = nullptr);
+ static void run_batch_update(vector<ID_TUPLE> id_tuples, TYPE_TRIPLE_NUM _triple_num, unsigned &update_num, UPDATE_TYPE type, shared_ptr<Transaction> txn = nullptr);
+
bool sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file);
bool sub2id_pre2id_obj2id_RDFintoSignature(const string _rdf_file,const string _error_log);
//bool literal2id_RDFintoSignature(const string _rdf_file, int** _p_id_tuples, TYPE_TRIPLE_NUM _id_tuples_max);

old_path: makefile

new_path: makefile

diff:
@@ -82,7 +82,7 @@ kvstoreobj = $(objdir)KVstore.o $(sitreeobj) $(istreeobj) $(ivtreeobj) $(ivarray
utilobj = $(objdir)Util.o $(objdir)Bstr.o $(objdir)Stream.o $(objdir)Triple.o $(objdir)BloomFilter.o $(objdir)VList.o \
$(objdir)EvalMultitypeValue.o $(objdir)IDTriple.o $(objdir)Version.o $(objdir)Transaction.o $(objdir)Latch.o $(objdir)IPWhiteList.o \
- $(objdir)IPBlackList.o
+ $(objdir)IPBlackList.o $(objdir)SpinLock.o $(objdir)GraphLock.o
queryobj = $(objdir)SPARQLquery.o $(objdir)BasicQuery.o $(objdir)ResultSet.o $(objdir)IDList.o \
$(objdir)Varset.o $(objdir)QueryTree.o $(objdir)TempResult.o $(objdir)QueryCache.o $(objdir)GeneralEvaluation.o \
@@ -373,7 +373,7 @@ $(objdir)IVArray.o: KVstore/IVArray/IVArray.cpp KVstore/IVArray/IVArray.h $(objd
$(objdir)IVBlockManager.o: KVstore/IVArray/IVBlockManager.cpp KVstore/IVArray/IVBlockManager.h
$(CC) $(CFLAGS) KVstore/IVArray/IVBlockManager.cpp -o $(objdir)IVBlockManager.o
-$(objdir)IVEntry.o: KVstore/IVArray/IVEntry.cpp KVstore/IVArray/IVEntry.h $(objdir)Version.o
+$(objdir)IVEntry.o: KVstore/IVArray/IVEntry.cpp KVstore/IVArray/IVEntry.h $(objdir)Version.o $(objdir)GraphLock.o
$(CC) $(CFLAGS) KVstore/IVArray/IVEntry.cpp -o $(objdir)IVEntry.o
#objects in ivarray/ end
@@ -489,6 +489,12 @@ $(objdir)EvalMultitypeValue.o: Util/EvalMultitypeValue.cpp Util/EvalMultitypeVal
$(objdir)Version.o: Util/Version.cpp Util/Version.h
$(CC) $(CFLAGS) Util/Version.cpp -o $(objdir)Version.o $(openmp)
+$(objdir)SpinLock.o: Util/SpinLock.h Util/SpinLock.cpp
+ $(CC) $(CFLAGS) Util/SpinLock.cpp -o $(objdir)SpinLock.o $(openmp)
+
+$(objdir)GraphLock.o: Util/GraphLock.h Util/GraphLock.cpp
+ $(CC) $(CFLAGS) Util/GraphLock.cpp -o $(objdir)GraphLock.o $(openmp)
+
$(objdir)Transaction.o: Util/Transaction.cpp Util/Transaction.h $(objdir)Util.o $(objdir)IDTriple.o
$(CC) $(CFLAGS) Util/Transaction.cpp $(inc) -o $(objdir)Transaction.o $(openmp)

original_message: fix the compile bugs raised by merge


message: remove out-of-date function and amend the function naming

diff:
@@ -5942,7 +5942,7 @@ Database::getFinalResult(SPARQLquery& _sparql_q, ResultSet& _result_set)
//garbage clean
void
-Database::version_clean(vector<unsigned> &sub_ids ,vector<unsigned>& obj_ids, vector<unsigned>& obj_literal_ids, vector<unsigned> &pre_ids)
+Database::VersionClean(vector<unsigned> &sub_ids ,vector<unsigned>& obj_ids, vector<unsigned>& obj_literal_ids, vector<unsigned> &pre_ids)
{
//vector<unsigned> sub_ids , obj_ids, obj_literal_ids, pre_ids;
(this->kvstore)->IVArrayVacuum(sub_ids, obj_ids, obj_literal_ids, pre_ids);
@@ -6045,7 +6045,7 @@ Database::version_clean(vector<unsigned> &sub_ids ,vector<unsigned>& obj_ids, ve
}
void
-Database::transaction_rollback(shared_ptr<Transaction> txn)
+Database::TransactionRollback(shared_ptr<Transaction> txn)
{
if((this->kvstore)->TransactionInvalid(txn) == false)
{
@@ -6055,7 +6055,7 @@ Database::transaction_rollback(shared_ptr<Transaction> txn)
}
void
-Database::transaction_commit(shared_ptr<Transaction> txn)
+Database::TransactionCommit(shared_ptr<Transaction> txn)
{
//cout << "transaction_commit ........" << endl;
if((this->kvstore)->ReleaseAllLocks(txn) == false)

diff:
@@ -100,9 +100,9 @@ public:
mutex& get_query_parse_lock();
//MVCC
- void transaction_rollback(shared_ptr<Transaction> txn);
- void transaction_commit(shared_ptr<Transaction> txn);
- void version_clean(vector<unsigned> &sub_ids ,vector<unsigned>& obj_ids, vector<unsigned>& obj_literal_ids, vector<unsigned> &pre_ids);
+ void TransactionRollback(shared_ptr<Transaction> txn);
+ void TransactionCommit(shared_ptr<Transaction> txn);
+ void VersionClean(vector<unsigned> &sub_ids ,vector<unsigned>& obj_ids, vector<unsigned>& obj_literal_ids, vector<unsigned> &pre_ids);
private:
string name;
string store_path;

diff:
@@ -177,7 +177,7 @@ int Txn_manager::Commit(txn_id_t TID)
txn_id_t CID = this->ArrangeCommitID();
txn->SetCommitID(CID);
if(db != nullptr)
- db->transaction_commit(txn);
+ db->TransactionCommit(txn);
else
{
cout << "error! database has been flushed or removed" << endl;
@@ -207,7 +207,7 @@ int Txn_manager::Abort(txn_id_t TID)
return -1;
}
if(db != nullptr)
- db->transaction_rollback(txn);
+ db->TransactionRollback(txn);
else
{
cout << "error! database has been flushed or removed" << endl;
@@ -294,7 +294,7 @@ void Txn_manager::Checkpoint()
//cout << "obj_ids.size(): " << obj_ids.size() << endl;
//cout << "obj_literal_ids.size(): " << obj_literal_ids.size() << endl;
if(db != nullptr)
- db->version_clean(sub_ids, obj_ids, obj_literal_ids, pre_ids);
+ db->VersionClean(sub_ids, obj_ids, obj_literal_ids, pre_ids);
checkpoint_lock.unlock();
}

diff:
@@ -38,8 +38,6 @@ private:
Latch checkpoint_lock;
Latch table_lock;
mutex DirtyKeys_lock;
- //not used
- mutex db_lock;
vector<IDSet> DirtyKeys;
atomic<int> committed_num = {0};
@@ -57,8 +55,6 @@ private:
inline void lock_log() { log_lock.lock(); }
inline void unlock_log() { log_lock.unlock(); }
- inline void lock_db() { db_lock.lock(); }
- inline void unlock_db() { db_lock.unlock(); }
void add_dirty_keys(shared_ptr<Transaction> txn);
public:

diff:
@@ -559,7 +559,7 @@ IVArray::search(unsigned _key, char *& _str, unsigned long & _len, VDataSet& Add
return false;
}
// try to read in main memory
- bool ret = array[_key].readVersion(AddSet, DelSet, txn, latched, is_firstread);
+ bool ret = array[_key].ReadVersion(AddSet, DelSet, txn, latched, is_firstread);
bool is_empty = AddSet.size() == 0 && DelSet.size() == 0;
if(ret == false) {
@@ -633,7 +633,7 @@ IVArray::remove(unsigned _key, VDataSet& delta, shared_ptr<Transaction> txn)
VDataSet addset;
addset.clear();
- bool ret = array[_key].writeVersion(addset, delta, txn);
+ bool ret = array[_key].WriteVersion(addset, delta, txn);
if(ret == false) txn->SetState(TransactionState::ABORTED);
ArrayUnlock();
return ret;
@@ -657,7 +657,7 @@ IVArray::insert(unsigned _key, VDataSet& delta, shared_ptr<Transaction> txn)
delset.clear();
//check if first insert
//array[_key].setDirtyFlag(true);
- int ret = array[_key].writeVersion(delta, delset, txn);
+ int ret = array[_key].WriteVersion(delta, delset, txn);
if(ret != 1) {
cerr << "write version failed!" << endl;
txn->SetState(TransactionState::ABORTED);
@@ -715,23 +715,11 @@ IVArray::TryExclusiveLatch(unsigned _key, shared_ptr<Transaction> txn, bool has_
}
//assert(_key < CurEntryNum);
- int ret = array[_key].getExclusiveLatch(txn, has_read);
+ int ret = array[_key].GetExclusiveLatch(txn, has_read);
ArrayUnlock();
return ret;
}
-/*
-bool
-IVArray::ReleaseExclusiveLock(unsigned _key, shared_ptr<Transaction> txn, bool has_read)
-{
- if (_key >= CurEntryNum || !array[_key].isUsed() )
- {
- return false;
- }
- return array[_key].releaseExlusiveLatch(txn, has_read);
-}
-*/
-
bool
IVArray::ReleaseLatch(unsigned _key, shared_ptr<Transaction> txn, IVEntry::LatchType type)
{
@@ -741,7 +729,7 @@ IVArray::ReleaseLatch(unsigned _key, shared_ptr<Transaction> txn, IVEntry::Latch
ArrayUnlock();
return false;
}
- bool ret = array[_key].unLatch(txn, type);
+ bool ret = array[_key].UnLatch(txn, type);
ArrayUnlock();
return ret;
}
@@ -756,7 +744,7 @@ IVArray::Rollback(unsigned _key, shared_ptr<Transaction> txn, bool has_read)
return false;
}
//bool delete_ret = array[_key].deleteUnCommittedVersion(txn);
- bool delete_ret = array[_key].invalidExlusiveLatch(txn, has_read);
+ bool delete_ret = array[_key].InvalidExlusiveLatch(txn, has_read);
//cerr << "delete_retdelete_ret TID:" << txn->GetTID() << " " << _key << " " << delete_ret << endl;
//bool unlock_ret = array[_key].releaseExlusiveLock(txn);
ArrayUnlock();
@@ -774,7 +762,7 @@ IVArray::CleanDirtyKey(unsigned _key)
ArrayUnlock();
return false;
}
- array[_key].cleanAllVersion();
+ array[_key].CleanAllVersion();
ArrayUnlock();
return true;

diff:
@@ -87,8 +87,6 @@ public:
//abort
//clean invalid version(release exclusive latch along) and release exclusive lock
bool Rollback(unsigned _key, shared_ptr<Transaction> txn, bool has_read );
- //unlock(abort)
- bool ReleaseExclusiveLock(unsigned _key, shared_ptr<Transaction> txn);
//garbage clean
bool CleanDirtyKey(unsigned _key) ;

diff:
@@ -199,8 +199,8 @@ void
IVEntry::getLatestVersion(TYPE_TXN_ID TID, VDataSet &addset, VDataSet &delset)
{
rwLatch.lockShared();
- int n = vList.size();
- for(int i = 0; i < n; i++)
+ unsigned n = vList.size();
+ for(unsigned i = 0; i < n - 1; i++)
{
vList[i]->get_version(addset, delset);
}
@@ -269,7 +269,7 @@ IVEntry::version_merge(VDataArray &addarray, VDataArray &delarray, VDataSet &Add
}
bool
-IVEntry::readVersion(VDataSet &AddSet, VDataSet &DelSet, shared_ptr<Transaction> txn, bool &latched, bool first_read)
+IVEntry::ReadVersion(VDataSet &AddSet, VDataSet &DelSet, shared_ptr<Transaction> txn, bool &latched, bool first_read)
{
if(txn->GetIsolationLevelType() == IsolationLevelType::READ_COMMITTED)
{
@@ -309,7 +309,7 @@ IVEntry::readVersion(VDataSet &AddSet, VDataSet &DelSet, shared_ptr<Transaction>
//txn aborts then delete its uncommitted version. Deleted version is always the latest version.
bool
-IVEntry::invalidExlusiveLatch(shared_ptr<Transaction> txn, bool has_read)
+IVEntry::InvalidExlusiveLatch(shared_ptr<Transaction> txn, bool has_read)
{
bool IS_SR = txn->GetIsolationLevelType() == IsolationLevelType::SERIALIZABLE;
auto TID = txn->GetTID();
@@ -341,7 +341,7 @@ IVEntry::invalidExlusiveLatch(shared_ptr<Transaction> txn, bool has_read)
}
bool
-IVEntry::unLatch(shared_ptr<Transaction> txn, LatchType latch_type)
+IVEntry::UnLatch(shared_ptr<Transaction> txn, LatchType latch_type)
{
bool IS_SR = (txn->GetIsolationLevelType() == IsolationLevelType::SERIALIZABLE);
auto TID = txn->GetTID();
@@ -394,7 +394,7 @@ IVEntry::clearVersionFlag()
}
void
-IVEntry::cleanAllVersion()
+IVEntry::CleanAllVersion()
{
rwLatch.lockExclusive();
vList.clear();
@@ -410,7 +410,7 @@ we don't need any lock in RC and SI here but only head version check
SR need exclusive lock to restrict the read operations!
*/
int
-IVEntry::getExclusiveLatch(shared_ptr<Transaction> txn, bool has_read)
+IVEntry::GetExclusiveLatch(shared_ptr<Transaction> txn, bool has_read)
{
auto TID = txn->GetTID();
shared_ptr<Version> new_version = make_shared<Version>(INVALID_TS, TID); //[-1, TID]
@@ -465,7 +465,7 @@ IVEntry::getExclusiveLatch(shared_ptr<Transaction> txn, bool has_read)
}
int
-IVEntry::writeVersion(VDataSet &AddSet, VDataSet &DelSet, shared_ptr<Transaction> txn)
+IVEntry::WriteVersion(VDataSet &AddSet, VDataSet &DelSet, shared_ptr<Transaction> txn)
{
//private version no need any lock here
rwLatch.lockShared();
@@ -478,10 +478,3 @@ IVEntry::writeVersion(VDataSet &AddSet, VDataSet &DelSet, shared_ptr<Transaction
rwLatch.unlock();
return 1;
}
-
-//abort only
-bool
-IVEntry::releaseExlusiveLatch(shared_ptr<Transaction> txn, bool has_read)
-{
- return this->invalidExlusiveLatch(txn, has_read);
-}
\ No newline at end of file

diff:
@@ -32,7 +32,7 @@ private:
//MVCC
atomic<bool> is_versioned;
- //Version List head point always point to the newest.
+ //Version List end point always point to the newest.
vector<shared_ptr<Version>> vList;
//protect Version list. TODO: use free-locked list for better concurrency
Latch rwLatch;
@@ -81,15 +81,14 @@ public:
void version_merge(VDataArray &addarray, VDataArray &delarray, VDataSet &AddSet, VDataSet &DelSet);
//get exclusive lock before update
- int getExclusiveLatch(shared_ptr<Transaction> txn, bool has_read);
- bool releaseExlusiveLatch(shared_ptr<Transaction> txn, bool has_read); //different from unlock
- bool invalidExlusiveLatch(shared_ptr<Transaction> txn, bool has_read); //undo
+ int GetExclusiveLatch(shared_ptr<Transaction> txn, bool has_read);
+ bool InvalidExlusiveLatch(shared_ptr<Transaction> txn, bool has_read);
- bool unLatch(shared_ptr<Transaction> txn, LatchType latch_type); //commit
+ bool UnLatch(shared_ptr<Transaction> txn, LatchType latch_type); //commit
- bool readVersion(VDataSet &AddSet, VDataSet &DelSet, shared_ptr<Transaction> txn, bool &latched, bool repeated_read = false ); //read
- int writeVersion(VDataSet &AddSet, VDataSet &DelSet, shared_ptr<Transaction> txn);
+ bool ReadVersion(VDataSet &AddSet, VDataSet &DelSet, shared_ptr<Transaction> txn, bool &latched, bool first_read = false ); //read
+ int WriteVersion(VDataSet &AddSet, VDataSet &DelSet, shared_ptr<Transaction> txn);
- void cleanAllVersion();
+ void CleanAllVersion();
~IVEntry();
};

diff:
@@ -27,8 +27,8 @@ private:
TransactionState state;
TYPE_TS timestamp;
- long int start_time;
- long int end_time;
+ unsigned long start_time;
+ unsigned long end_time;
//latch table
vector<IDSet> ReadSet; //shared latches
@@ -39,10 +39,11 @@ private:
vector<string> sparqls;
IsolationLevelType isolation;
+
int wait_lock_time;
int retry_times;
public:
- unsigned long long int update_num;
+ int update_num;
enum class IDType{SUBJECT, PREDICATE, OBJECT};
Transaction(Transaction const&) = delete;
Transaction(string name, TYPE_TS time, txn_id_t TID, IsolationLevelType _isolation = IsolationLevelType::SERIALIZABLE);
@@ -74,11 +75,6 @@ public:
void WriteSetDelete(IDTriple _Triple);
bool WriteSetFind(TYPE_ENTITY_LITERAL_ID _ID, Transaction::IDType _type);
- void AddSetInsert(IDTriple _Triple);
- void AddSetDelete(IDTriple _Triple);
- void DelSetInsert(IDTriple _Triple);
- void DelSetDelete(IDTriple _Triple);
-
//void DependedTXNSetInsert(TYPE_TXN_ID _TID);
//void DependedTXNSetDelete(TYPE_TXN_ID _TID);
@@ -92,6 +88,7 @@ public:
int get_wait_lock_time() { return this->wait_lock_time;}
int get_retry_times() { return this->retry_times; }
+ //DEBUG Function
void print_ReadSet();
void print_WriteSet();
void print_AddSet();

change_type: DELETE

old_path: api/socket/cpp/example/example

new_path: api/socket/cpp/example/example

diff: Binary files a/api/socket/cpp/example/example and /dev/null differ

diff: Binary files a/scripts/run_transaction and b/scripts/run_transaction differ

diff: Binary files a/scripts/transaction_test and b/scripts/transaction_test differ

old_path: scripts/workload

new_path: scripts/workload

diff: Binary files a/scripts/workload and b/scripts/workload differ


message:
Bugfix of transaction
1) fix bugs of transaction api
2) fix a bug of transaction under read committed level

diff:
@@ -203,10 +203,12 @@ IVEntry::getLatestVersion(TYPE_TXN_ID TID, VDataSet &addset, VDataSet &delset)
for(unsigned i = 0; i < n - 1; i++)
{
vList[i]->get_version(addset, delset);
+ cout << "begin_ts" << vList[i]->get_begin_ts() << " end_ts" << vList[i]->get_end_ts() << endl;
}
- if((vList[n-1]->get_begin_ts() == INVALID_TS && vList[n-1]->get_end_ts() == TID) || (vList[n-1]->get_end_ts() == INVALID_TS && vList[n-1]->get_begin_ts() < TID)) //private version or committed version
+ if((vList[n-1]->get_begin_ts() == INVALID_TS && vList[n-1]->get_end_ts() == TID) || (vList[n-1]->get_end_ts() == INVALID_TS)) //private version or committed version
{
vList[n-1]->get_version(addset, delset);
+ cout << "begin_ts" << vList[n-1]->get_begin_ts() << " end_ts" << vList[n-1]->get_end_ts() << endl;
}
rwLatch.unlock();
}
@@ -367,6 +369,7 @@ IVEntry::UnLatch(shared_ptr<Transaction> txn, LatchType latch_type)
}
//update the version info(unlock)
//assert(vList[k-1]->get_end_ts() == TID);
+ cout << "cid..................." << cid << endl;
vList[k-1]->set_begin_ts(cid);
vList[k-1]->set_end_ts(INVALID_TS);
vList[k-2]->set_end_ts(cid);

diff:
@@ -571,13 +571,13 @@ std::string GstoreConnector::tquery(std::string db_name, std::string TID, std::s
std::string res;
if (request_type == "GET")
{
- std::string strUrl = this->Url + "/?operation=tquery&db_name=" + db_name + "&username=" + this->username + "&password=" + this->password + "&TID=" + TID + "&sparql=" + sparql;
+ std::string strUrl = this->Url + "/?operation=tquery&db_name=" + db_name + "&username=" + this->username + "&password=" + this->password + "&tid=" + TID + "&sparql=" + sparql;
int ret = this->Get(strUrl, res);
}
else if (request_type == "POST")
{
std::string strUrl = this->Url + "/export";
- std::string strPost = "{\"db_name\": \"" + db_name + "\", \"username\": \"" + this->username + "\", \"password\": \"" + this->password + "\", \"TID\": \"" + TID+ "\", \"sparql\": \"" + sparql + "\"}";
+ std::string strPost = "{\"db_name\": \"" + db_name + "\", \"username\": \"" + this->username + "\", \"password\": \"" + this->password + "\", \"tid\": \"" + TID+ "\", \"sparql\": \"" + sparql + "\"}";
int ret = this->Post(strUrl, strPost, res);
}
return res;
@@ -588,13 +588,13 @@ std::string GstoreConnector::commit(std::string db_name, std::string TID, std::s
std::string res;
if (request_type == "GET")
{
- std::string strUrl = this->Url + "/?operation=commit&db_name=" + db_name + "&username=" + this->username + "&password=" + this->password + "&TID=" + TID;
+ std::string strUrl = this->Url + "/?operation=commit&db_name=" + db_name + "&username=" + this->username + "&password=" + this->password + "&tid=" + TID;
int ret = this->Get(strUrl, res);
}
else if (request_type == "POST")
{
std::string strUrl = this->Url + "/export";
- std::string strPost = "{\"db_name\": \"" + db_name + "\", \"username\": \"" + this->username + "\", \"password\": \"" + this->password + "\", \"TID\": \"" + TID + "\"}";
+ std::string strPost = "{\"db_name\": \"" + db_name + "\", \"username\": \"" + this->username + "\", \"password\": \"" + this->password + "\", \"tid\": \"" + TID + "\"}";
int ret = this->Post(strUrl, strPost, res);
}
return res;
@@ -605,13 +605,13 @@ std::string GstoreConnector::rollback(std::string db_name, std::string TID, std:
std::string res;
if (request_type == "GET")
{
- std::string strUrl = this->Url + "/?operation=rollback&db_name=" + db_name + "&username=" + this->username + "&password=" + this->password + "&TID=" + TID;
+ std::string strUrl = this->Url + "/?operation=rollback&db_name=" + db_name + "&username=" + this->username + "&password=" + this->password + "&tid=" + TID;
int ret = this->Get(strUrl, res);
}
else if (request_type == "POST")
{
std::string strUrl = this->Url + "/rollback";
- std::string strPost = "{\"db_name\": \"" + db_name + "\", \"username\": \"" + this->username + "\", \"password\": \"" + this->password + "\", \"TID\": \"" + TID + "\"}";
+ std::string strPost = "{\"db_name\": \"" + db_name + "\", \"username\": \"" + this->username + "\", \"password\": \"" + this->password + "\", \"tid\": \"" + TID + "\"}";
int ret = this->Post(strUrl, strPost, res);
}
return res;


message:
Fix of Txn Log
TID is replaced by combination of time and TID

diff:
@@ -3015,7 +3015,8 @@ void begin_thread_new(const shared_ptr<HttpServer::Response>& response,string db
txn_id_t TID = txn_m->Begin(static_cast<IsolationLevelType>(level));
cout <<"Transcation Id:"<< to_string(TID) << endl;
cout << to_string(txn_m->Get_Transaction(TID)->GetStartTime()) << endl;
- Util::add_transactionlog(db_name, username, to_string(TID), to_string(txn_m->Get_Transaction(TID)->GetStartTime()), "RUNNING", "INF");
+ string Time_TID = Util::get_date_time() + " " + to_string(TID);
+ Util::add_transactionlog(db_name, username, Time_TID, to_string(txn_m->Get_Transaction(TID)->GetStartTime()), "RUNNING", "INF");
if (TID == INVALID_ID)
{
error = "transaction begin failed.";
@@ -3254,7 +3255,8 @@ void commit_thread_new(const shared_ptr<HttpServer::Response>& response,string d
}
else
{
- Util::update_transactionlog(to_string(TID), "COMMITED", to_string(txn_m->Get_Transaction(TID)->GetEndTime()));
+ string Time_TID = Util::get_date_time() + " " + to_string(TID);
+ Util::update_transactionlog(Time_TID, "COMMITED", to_string(txn_m->Get_Transaction(TID)->GetEndTime()));
string success = "transaction commit success. TID: " + TID_s;
sendResponseMsg(0,success,response);
return;
@@ -3343,7 +3345,8 @@ void rollback_thread_new(const shared_ptr<HttpServer::Response>& response,string
}
else
{
- Util::update_transactionlog(to_string(TID), "ROLLBACK", to_string(txn_m->Get_Transaction(TID)->GetEndTime()));
+ string Time_TID = Util::get_date_time() + " " + to_string(TID);
+ Util::update_transactionlog(Time_TID, "ROLLBACK", to_string(txn_m->Get_Transaction(TID)->GetEndTime()));
string success = "transaction rollback success. TID: " + TID_s;
sendResponseMsg(0,success,response);
return;


message: fix bug of checkpoint

diff:
@@ -3466,7 +3466,18 @@ void checkpoint_thread_new(const shared_ptr<HttpServer::Response>& response,stri
}
else
{
+ pthread_rwlock_rdlock(&txn_m_lock);
+ if (txn_managers.find(db_name) == txn_managers.end())
+ {
+ error = "Database transaction manager error.";
+ sendResponseMsg(1004,error,response);
+ pthread_rwlock_unlock(&txn_m_lock);
+ return;
+ }
+ shared_ptr<Txn_manager> txn_m = txn_managers[db_name];
+ pthread_rwlock_unlock(&txn_m_lock);
+ txn_m->Checkpoint();
current_database->save();
//NOTICE: this info is in header
string success = "Database saved successfully.";

original_message: fix bug of checkpoint


message: fix a bug of garbage collection

diff:
@@ -5802,7 +5802,7 @@ KVstore::o2values_literal_vacuum(vector<unsigned>& obj_literal_ids, shared_ptr<T
//cout << "_len: " << _len << endl;
//for(int i = 0; i < _len / sizeof(unsigned); i++) cout << _tmp[i] << " ";
//cout << endl;
- this->clean_dirty_key(this->objID2values, _obj_id);
+ this->clean_dirty_key(this->objID2values_literal, _obj_id - Util::LITERAL_FIRST_ID);
if(_len == 0) {
if(base_empty == false)
{

diff:
@@ -62,6 +62,23 @@ void Bug4SItree(Database *_db)
}
}
+void Bug20220408(Txn_manager &txn_m)
+{
+ string insert = "insert data {<zhangzhe> <name> \"zhangzhe\"}";
+ string remove = "delete data {<zhangzhe> <name> \"zhangzhe\"}";
+ string query = "select ?x where {<zhangzhe> ?p ?x}";
+ string res;
+ txn_id_t TID = txn_m.Begin();
+ txn_m.Query(TID, remove, res);
+ txn_m.Query(TID, query, res);
+ txn_m.Commit(TID);
+ txn_m.Checkpoint();
+ TID = txn_m.Begin();
+ txn_m.Query(TID, insert, res);
+ txn_m.Query(TID, query, res);
+ txn_m.Commit(TID);
+ txn_m.Checkpoint();
+}
int main(int argc, char* argv[])
{
@@ -71,5 +88,6 @@ int main(int argc, char* argv[])
Database _db(db_folder);
_db.load();
- Bug4SItree(&_db);
+ Txn_manager txn_m(&_db, "test");
+ Bug20220408(txn_m);
}
\ No newline at end of file

original_message: fix a bug of garbage collection


message: docs(concatAll): fix map import

diff:
@@ -32,7 +32,7 @@ export function concatAll<R>(): OperatorFunction<any, R>;
* For each click event, tick every second from 0 to 3, with no concurrency
* ```javascript
* import { fromEvent, interval } from 'rxjs';
- * import { ma, take, concatAll } from 'rxjs/operators';
+ * import { map, take, concatAll } from 'rxjs/operators';
*
* const clicks = fromEvent(document, 'click');
* const higherOrder = clicks.pipe(

license: Apache License 2.0

repo: reactivex/rxjs

original_message: docs(concatAll): fix map import (#4535)


message: docs(windowTime): fix examples

diff:
@@ -34,12 +34,12 @@ import { OperatorFunction, SchedulerLike, SchedulerAction } from '../types';
* In every window of 1 second each, emit at most 2 click events
* ```javascript
* import { fromEvent } from 'rxjs';
- * import { windowTime, map, mergeAll } from 'rxjs/operators';
+ * import { windowTime, map, mergeAll, take } from 'rxjs/operators';
*
* const clicks = fromEvent(document, 'click');
* const result = clicks.pipe(
* windowTime(1000),
- * map(win => win.take(2)), // each window has at most 2 emissions
+ * map(win => win.pipe(take(2))), // each window has at most 2 emissions
* mergeAll(), // flatten the Observable-of-Observables
* );
* result.subscribe(x => console.log(x));
@@ -48,12 +48,12 @@ import { OperatorFunction, SchedulerLike, SchedulerAction } from '../types';
* Every 5 seconds start a window 1 second long, and emit at most 2 click events per window
* ```javascript
* import { fromEvent } from 'rxjs';
- * import { windowTime, map, mergeAll } from 'rxjs/operators';
+ * import { windowTime, map, mergeAll, take } from 'rxjs/operators';
*
* const clicks = fromEvent(document, 'click');
* const result = clicks.pipe(
* windowTime(1000, 5000),
- * map(win => win.take(2)), // each window has at most 2 emissions
+ * map(win => win.pipe(take(2))), // each window has at most 2 emissions
* mergeAll(), // flatten the Observable-of-Observables
* );
* result.subscribe(x => console.log(x));

original_message: docs(windowTime): fix examples (#4547)


message: docs(windowWhen): fix example

diff:
@@ -28,13 +28,13 @@ import { OperatorFunction } from '../types';
* Emit only the first two clicks events in every window of [1-5] random seconds
* ```javascript
* import { fromEvent, interval } from 'rxjs';
- * import { windowWhen, map, mergeAll } from 'rxjs/operators';
+ * import { windowWhen, map, mergeAll, take } from 'rxjs/operators';
*
* const clicks = fromEvent(document, 'click');
* const result = clicks.pipe(
* windowWhen(() => interval(1000 + Math.random() * 4000)),
* map(win => win.pipe(take(2))), // each window has at most 2 emissions
- * mergeAll(), // flatten the Observable-of-Observables
+ * mergeAll() // flatten the Observable-of-Observables
* );
* result.subscribe(x => console.log(x));
* ```

original_message: docs(windowWhen): fix example (#4548)


message: docs(windowToggle): fix example

diff:
@@ -27,14 +27,14 @@ import { OperatorFunction } from '../types';
* ## Example
* Every other second, emit the click events from the next 500ms
* ```javascript
- * import { fromEvent, interval } from 'rxjs';
+ * import { fromEvent, interval, EMPTY } from 'rxjs';
* import { windowToggle, mergeAll } from 'rxjs/operators';
*
* const clicks = fromEvent(document, 'click');
* const openings = interval(1000);
* const result = clicks.pipe(
- * windowToggle(openings, i => i % 2 ? interval(500) : empty()),
- * mergeAll(),
+ * windowToggle(openings, i => i % 2 ? interval(500) : EMPTY),
+ * mergeAll()
* );
* result.subscribe(x => console.log(x));
* ```

original_message: docs(windowToggle): fix example (#4549)


message: docs(auditTime): fix docs typo

diff:
@@ -7,7 +7,7 @@ import { MonoTypeOperatorFunction, SchedulerLike } from '../types';
* Ignores source values for `duration` milliseconds, then emits the most recent
* value from the source Observable, then repeats this process.
*
- * When it sees a source values, it ignores that plus
+ * When it sees a source value, it ignores that plus
* the next ones for `duration` milliseconds, and then it emits the most recent
* value from the source.
*

original_message: docs(auditTime): fix docs typo (#4550)


diff:
@@ -26,13 +26,13 @@ import { OperatorFunction, SubscribableOrPromise } from '../types';
* Every other second, emit the click events from the next 500ms
*
* ```javascript
- * import { fromEvent, interval, empty } from 'rxjs';
+ * import { fromEvent, interval, EMPTY } from 'rxjs';
* import { bufferToggle } from 'rxjs/operators';
*
* const clicks = fromEvent(document, 'click');
* const openings = interval(1000);
* const buffered = clicks.pipe(bufferToggle(openings, i =>
- * i % 2 ? interval(500) : empty()
+ * i % 2 ? interval(500) : EMPTY
* ));
* buffered.subscribe(x => console.log(x));
* ```

original_message: docs(bufferToggle): use non-deprecated syntax in example (#4552)


message: docs(buffer): fix example

diff:
@@ -28,8 +28,8 @@ import { OperatorFunction } from '../types';
* import { buffer } from 'rxjs/operators';
*
* const clicks = fromEvent(document, 'click');
- * const interval = interval(1000);
- * const buffered = interval.pipe(buffer(clicks));
+ * const intervalEvents = interval(1000);
+ * const buffered = intervalEvents.pipe(buffer(clicks));
* buffered.subscribe(x => console.log(x));
* ```
*

original_message: docs(buffer): fix example (#4551)


message: docs(window): fix import syntax

diff:
@@ -26,7 +26,7 @@ import { Operator } from '../Operator';
* In every window of 1 second each, emit at most 2 click events
* ```javascript
* import { fromEvent, interval } from 'rxjs';
- * import { window, mergeAll, map take } from 'rxjs/operators';
+ * import { window, mergeAll, map, take } from 'rxjs/operators';
*
* const clicks = fromEvent(document, 'click');
* const sec = interval(1000);

original_message: docs(window): fix import syntax (#4545)


message:
docs(publish): fix example
Fixes unterminated comment error.
Adds missing `tap` import.
Tidies formatting.
Converts language identifier to `javascript` to match other examples.

diff:
@@ -21,36 +21,34 @@ export function publish<T>(selector: MonoTypeOperatorFunction<T>): MonoTypeOpera
* ## Examples
* Make source$ hot by applying publish operator, then merge each inner observable into a single one
* and subscribe.
- * ```typescript
+ * ```javascript
* import { of, zip, interval, merge } from "rxjs";
- * import { map, publish } from "rxjs/operators";
+ * import { map, publish, tap } from "rxjs/operators";
*
- * const source$ = zip(
- * interval(2000),
- * of(1, 2, 3, 4, 5, 6, 7, 8, 9),
- * ).pipe(
+ * const source$ = zip(interval(2000), of(1, 2, 3, 4, 5, 6, 7, 8, 9)).pipe(
* map(values => values[1])
* );
*
- * source$.pipe(
- * publish(multicasted$ => {
- * return merge(
+ * source$
+ * .pipe(
+ * publish(multicasted$ =>
+ * merge(
* multicasted$.pipe(tap(x => console.log('Stream 1:', x))),
* multicasted$.pipe(tap(x => console.log('Stream 2:', x))),
* multicasted$.pipe(tap(x => console.log('Stream 3:', x))),
- * );
- * })).subscribe();
- *
- /* Results every two seconds
- * Stream 1: 1
- * Stream 2: 1
- * Stream 3: 1
- *
- * ...
+ * )
+ * )
+ * )
+ * .subscribe();
*
- * Stream 1: 9
- * Stream 2: 9
- * Stream 3: 9
+ * // Results every two seconds
+ * // Stream 1: 1
+ * // Stream 2: 1
+ * // Stream 3: 1
+ * // ...
+ * // Stream 1: 9
+ * // Stream 2: 9
+ * // Stream 3: 9
* ```
*
* @param {Function} [selector] - Optional selector function which can use the multicasted source sequence as many times

original_message:
docs(publish): fix example (#4557)

- Fixes unterminated comment error.
- Adds missing `tap` import.
- Tidies formatting.
- Converts language identifier to `javascript` to match other examples.


message: docs(windowCount): fix examples

diff:
@@ -26,13 +26,13 @@ import { OperatorFunction } from '../types';
* Ignore every 3rd click event, starting from the first one
* ```javascript
* import { fromEvent } from 'rxjs';
- * import { windowCount, map, mergeAll } from 'rxjs/operators';
+ * import { windowCount, map, mergeAll, skip } from 'rxjs/operators';
*
* const clicks = fromEvent(document, 'click');
* const result = clicks.pipe(
- * windowCount(3)),
- * map(win => win.skip(1)), // skip first of every 3 clicks
- * mergeAll(), // flatten the Observable-of-Observables
+ * windowCount(3),
+ * map(win => win.pipe(skip(1))), // skip first of every 3 clicks
+ * mergeAll() // flatten the Observable-of-Observables
* );
* result.subscribe(x => console.log(x));
* ```

original_message: docs(windowCount): fix examples (#4546)


message: docs(observable): fix examples

diff:
@@ -121,20 +121,24 @@ export class Observable<T> implements Subscribable<T> {
* ## Example
* ### Subscribe with an Observer
* ```javascript
+ * import { of } from 'rxjs';
+ *
* const sumObserver = {
* sum: 0,
* next(value) {
* console.log('Adding: ' + value);
* this.sum = this.sum + value;
* },
- * error() { // We actually could just remove this method,
- * }, // since we do not really care about errors right now.
+ * error() {
+ * // We actually could just remove this method,
+ * // since we do not really care about errors right now.
+ * },
* complete() {
* console.log('Sum equals: ' + this.sum);
* }
* };
*
- * Rx.Observable.of(1, 2, 3) // Synchronously emits 1, 2, 3 and then completes.
+ * of(1, 2, 3) // Synchronously emits 1, 2, 3 and then completes.
* .subscribe(sumObserver);
*
* // Logs:
@@ -146,18 +150,17 @@ export class Observable<T> implements Subscribable<T> {
*
* ### Subscribe with functions
* ```javascript
+ * import { of } from 'rxjs';
+ *
* let sum = 0;
*
- * Rx.Observable.of(1, 2, 3)
- * .subscribe(
- * function(value) {
+ * of(1, 2, 3).subscribe(
+ * value => {
* console.log('Adding: ' + value);
* sum = sum + value;
* },
* undefined,
- * function() {
- * console.log('Sum equals: ' + sum);
- * }
+ * () => console.log('Sum equals: ' + sum)
* );
*
* // Logs:
@@ -169,12 +172,16 @@ export class Observable<T> implements Subscribable<T> {
*
* ### Cancel a subscription
* ```javascript
- * const subscription = Rx.Observable.interval(1000).subscribe(
+ * import { interval } from 'rxjs';
+ *
+ * const subscription = interval(1000).subscribe(
* num => console.log(num),
* undefined,
- * () => console.log('completed!') // Will not be called, even
- * ); // when cancelling subscription
- *
+ * () => {
+ * // Will not be called, even when cancelling subscription.
+ * console.log('completed!');
+ * }
+ * );
*
* setTimeout(() => {
* subscription.unsubscribe();

original_message: docs(observable): fix examples (#4562)


message: docs(concat): tidy example

diff:
@@ -91,7 +91,7 @@ export function concat<R>(...observables: (ObservableInput<any> | SchedulerLike)
* import { take } from 'rxjs/operators';
*
* const timer = interval(1000).pipe(take(2));
- * *
+ *
* concat(timer, timer) // concatenating the same Observable!
* .subscribe(
* value => console.log(value),

original_message: docs(concat): tidy example (#4620)


message:
docs(groupBy): fix examples
Fixes a few glitches so the examples are runnable.
This also includes stripping the type definitions since the example
code blocks specify `javascript`, and the other examples I've seen so
far don't seem to include these.

diff:
@@ -38,12 +38,7 @@ export function groupBy<T, K, R>(keySelector: (value: T) => K, elementSelector?:
* import { of } from 'rxjs';
* import { mergeMap, groupBy, reduce } from 'rxjs/operators';
*
- * interface Obj {
- * id: number,
- * name: string,
- * }
- *
- * of<Obj>(
+ * of(
* {id: 1, name: 'JavaScript'},
* {id: 2, name: 'Parcel'},
* {id: 2, name: 'webpack'},
@@ -69,18 +64,21 @@ export function groupBy<T, K, R>(keySelector: (value: T) => K, elementSelector?:
*
* ```javascript
* import { of } from 'rxjs';
- * import { mergeMap, groupBy, map, reduce } from 'rxjs/operators';
+ * import { groupBy, map, mergeMap, reduce } from 'rxjs/operators';
*
- * of<Obj>(
+ * of(
* { id: 1, name: 'JavaScript' },
* { id: 2, name: 'Parcel' },
* { id: 2, name: 'webpack' },
- * {id: 1, name: 'TypeScript'}
+ * { id: 1, name: 'TypeScript' },
* { id: 3, name: 'TSLint' }
- * ).pipe(
+ * )
+ * .pipe(
* groupBy(p => p.id, p => p.name),
- * mergeMap( (group$) => group$.pipe(reduce((acc, cur) => [...acc, cur], ["" + group$.key]))),
- * map(arr => ({'id': parseInt(arr[0]), 'values': arr.slice(1)})),
+ * mergeMap(group$ =>
+ * group$.pipe(reduce((acc, cur) => [...acc, cur], [`${group$.key}`]))
+ * ),
+ * map(arr => ({ id: parseInt(arr[0], 10), values: arr.slice(1) }))
* )
* .subscribe(p => console.log(p));
*

original_message:
docs(groupBy): fix examples (#4554)

Fixes a few glitches so the examples are runnable.

This also includes stripping the type definitions since the example
code blocks specify `javascript`, and the other examples I've seen so
far don't seem to include these.


message: docs(forkJoin): fix example

diff:
@@ -110,7 +110,7 @@ export function forkJoin<T>(...sources: ObservableInput<T>[]): Observable<T[]>;
* ### Use forkJoin with project function
* ```javascript
* import { forkJoin, interval } from 'rxjs';
- * import { take } from 'rxjs/operators';
+ * import { map, take } from 'rxjs/operators';
*
* const observable = forkJoin(
* interval(1000).pipe(take(3)), // emit 0, 1, 2 every second and complete

original_message: docs(forkJoin): fix example (#4628)


message: docs(exhaustMap): fix example

diff:
@@ -38,12 +38,12 @@ export function exhaustMap<T, I, R>(project: (value: T, index: number) => Observ
* ## Example
* Run a finite timer for each click, only if there is no currently active timer
* ```ts
- * import { fromEvent, } from 'rxjs';
+ * import { fromEvent, interval } from 'rxjs';
* import { exhaustMap, take } from 'rxjs/operators';
*
* const clicks = fromEvent(document, 'click');
* const result = clicks.pipe(
- * exhaustMap((ev) => interval(1000).pipe(take(5))),
+ * exhaustMap(ev => interval(1000).pipe(take(5)))
* );
* result.subscribe(x => console.log(x));
* ```

original_message: docs(exhaustMap): fix example (#4625)


message:
provider batch get transactioninfo
* provider batch get transactioninfo
provider batch get transactioninfo
* provider batch get transactioninfo
provider batch get transactioninfo
* get batch transaction infos
get batch transaction infos
* provider get transaction infos
fmt code

diff:
@@ -68,6 +68,13 @@ pub trait ChainReader {
fn verify(&self, block: Block) -> Result<VerifiedBlock>;
/// Execute block and verify it execute state, and save result base current chain, but do not change current chain.
fn execute(&self, block: VerifiedBlock) -> Result<ExecutedBlock>;
+ /// Get chain transaction infos
+ fn get_txn_infos(
+ &self,
+ start_index: u64,
+ reverse: bool,
+ max_size: u64,
+ ) -> Result<Vec<BlockTransactionInfo>>;
}
pub trait ChainWriter {

diff:
@@ -47,6 +47,11 @@ pub enum ChainRequest {
},
GetBlocks(Vec<HashValue>),
GetHeaders(Vec<HashValue>),
+ GetTransactionInfos {
+ start_index: u64,
+ reverse: bool,
+ max_size: u64,
+ },
}
impl ServiceRequest for ChainRequest {

diff:
@@ -48,6 +48,13 @@ pub trait ReadableChainService {
reverse: bool,
max_size: u64,
) -> Result<Vec<HashValue>>;
+
+ fn get_txn_infos(
+ &self,
+ start_index: u64,
+ reverse: bool,
+ max_size: u64,
+ ) -> Result<Vec<BlockTransactionInfo>>;
}
/// Writeable block chain service trait
@@ -99,6 +106,12 @@ pub trait ChainAsyncService:
reverse: bool,
max_size: u64,
) -> Result<Vec<HashValue>>;
+ async fn get_txn_infos(
+ &self,
+ start_index: u64,
+ reverse: bool,
+ max_size: u64,
+ ) -> Result<Vec<BlockTransactionInfo>>;
}
#[async_trait::async_trait]
@@ -347,4 +360,24 @@ where
bail!("get_block_ids invalid response")
}
}
+
+ async fn get_txn_infos(
+ &self,
+ start_index: u64,
+ reverse: bool,
+ max_size: u64,
+ ) -> Result<Vec<BlockTransactionInfo>> {
+ let response = self
+ .send(ChainRequest::GetTransactionInfos {
+ start_index,
+ reverse,
+ max_size,
+ })
+ .await??;
+ if let ChainResponse::TransactionInfos(tx_infos) = response {
+ Ok(tx_infos)
+ } else {
+ bail!("get txn infos error")
+ }
+ }
}

diff:
@@ -214,6 +214,15 @@ impl ServiceHandler<Self, ChainRequest> for ChainReaderService {
ChainRequest::GetHeaders(ids) => {
Ok(ChainResponse::BlockHeaderVec(self.inner.get_headers(ids)?))
}
+ ChainRequest::GetTransactionInfos {
+ start_index,
+ reverse,
+ max_size,
+ } => Ok(ChainResponse::TransactionInfos(self.inner.get_txn_infos(
+ start_index,
+ reverse,
+ max_size,
+ )?)),
}
}
}
@@ -353,6 +362,15 @@ impl ReadableChainService for ChainReaderServiceInner {
) -> Result<Vec<HashValue>> {
self.main.get_block_ids(start_number, reverse, max_size)
}
+
+ fn get_txn_infos(
+ &self,
+ start_index: u64,
+ reverse: bool,
+ max_size: u64,
+ ) -> Result<Vec<BlockTransactionInfo>> {
+ self.main.get_txn_infos(start_index, reverse, max_size)
+ }
}
#[cfg(test)]

diff:
@@ -721,6 +721,30 @@ impl ChainReader for BlockChain {
verified_block.0,
)
}
+
+ fn get_txn_infos(
+ &self,
+ start_index: u64,
+ reverse: bool,
+ max_size: u64,
+ ) -> Result<Vec<BlockTransactionInfo>> {
+ let chain_header = self.current_header();
+ let hash_list = self
+ .txn_accumulator
+ .get_leaves(start_index, reverse, max_size)?;
+ let mut infos = vec![];
+ for hash in hash_list {
+ let info = self.storage.get_transaction_info(hash)?.ok_or_else(|| {
+ anyhow::anyhow!(format!(
+ "cannot find hash({}) on main chain(head: {})",
+ hash,
+ chain_header.id()
+ ))
+ })?;
+ infos.push(info);
+ }
+ Ok(infos)
+ }
}
impl BlockChain {

diff:
+// Copyright (c) The Starcoin Core Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+use crate::cli_state::CliState;
+use crate::StarcoinOpt;
+use anyhow::Result;
+use scmd::{CommandAction, ExecContext};
+use starcoin_rpc_api::types::TransactionInfoView;
+use structopt::StructOpt;
+
+/// Get transaction infos list
+#[derive(Debug, StructOpt)]
+#[structopt(name = "get-txn-info-list", alias = "get_txn_info_list")]
+pub struct GetTransactionInfoListOpt {
+ /// start_index
+ #[structopt(name = "start_index", long, short = "s")]
+ start_index: u64,
+
+ #[structopt(name = "reverse", long, short = "r")]
+ reverse: Option<bool>,
+
+ #[structopt(name = "count", long, short = "c")]
+ count: u64,
+}
+
+pub struct GetTransactionInfoListCommand;
+
+impl CommandAction for GetTransactionInfoListCommand {
+ type State = CliState;
+ type GlobalOpt = StarcoinOpt;
+ type Opt = GetTransactionInfoListOpt;
+ type ReturnItem = Vec<TransactionInfoView>;
+
+ fn run(
+ &self,
+ ctx: &ExecContext<Self::State, Self::GlobalOpt, Self::Opt>,
+ ) -> Result<Self::ReturnItem> {
+ let client = ctx.state().client();
+ let opt = ctx.opt();
+ let txn_infos = client.chain_get_transaction_infos(
+ opt.start_index,
+ opt.reverse.unwrap_or(false),
+ opt.count,
+ )?;
+ Ok(txn_infos)
+ }
+}

diff:
@@ -6,6 +6,7 @@ mod get_block_cmd;
mod get_events_cmd;
mod get_txn_cmd;
mod get_txn_info_cmd;
+mod get_txn_info_list_cmd;
mod get_txn_infos_cmd;
mod info_cmd;
mod list_block_cmd;
@@ -15,6 +16,7 @@ pub use get_block_cmd::*;
pub use get_events_cmd::*;
pub use get_txn_cmd::*;
pub use get_txn_info_cmd::*;
+pub use get_txn_info_list_cmd::*;
pub use get_txn_infos_cmd::*;
pub use info_cmd::*;
pub use list_block_cmd::*;

diff:
@@ -100,7 +100,8 @@ pub fn add_command(
.subcommand(chain::GetTxnInfosCommand)
.subcommand(chain::GetTransactionInfoCommand)
.subcommand(chain::GetEventsCommand)
- .subcommand(chain::EpochInfoCommand),
+ .subcommand(chain::EpochInfoCommand)
+ .subcommand(chain::GetTransactionInfoListCommand),
)
.command(
Command::with_name("txpool")

diff:
@@ -54,7 +54,7 @@ pub trait ChainApi {
transaction_hash: HashValue,
option: Option<GetTransactionOption>,
) -> FutureResult<Option<TransactionView>>;
- /// Get chain transactions
+ /// Get chain transaction info
#[rpc(name = "chain.get_transaction_info")]
fn get_transaction_info(
&self,
@@ -88,6 +88,13 @@ pub trait ChainApi {
/// Get headers by ids.
#[rpc(name = "chain.get_headers")]
fn get_headers(&self, ids: Vec<HashValue>) -> FutureResult<Vec<BlockHeaderView>>;
+ #[rpc(name = "chain.get_transaction_infos")]
+ fn get_transaction_infos(
+ &self,
+ start_index: u64,
+ reverse: bool,
+ max_size: u64,
+ ) -> FutureResult<Vec<TransactionInfoView>>;
}
#[derive(Copy, Clone, Default, Serialize, Deserialize, JsonSchema)]

diff:
@@ -753,6 +753,20 @@ impl RpcClient {
.map_err(map_err)
}
+ pub fn chain_get_transaction_infos(
+ &self,
+ start_index: u64,
+ reverse: bool,
+ max_size: u64,
+ ) -> anyhow::Result<Vec<TransactionInfoView>> {
+ self.call_rpc_blocking(|inner| {
+ inner
+ .chain_client
+ .get_transaction_infos(start_index, reverse, max_size)
+ })
+ .map_err(map_err)
+ }
+
pub fn dry_run(&self, txn: DryRunTransactionRequest) -> anyhow::Result<DryRunOutputView> {
self.call_rpc_blocking(|inner| inner.contract_client.dry_run(txn))
.map_err(map_err)

diff:
}
}
}
+ },
+ {
+ "name": "chain.get_transaction_infos",
+ "params": [
+ {
+ "name": "start_index",
+ "schema": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "uint64",
+ "type": "integer",
+ "format": "uint64",
+ "minimum": 0.0
+ }
+ },
+ {
+ "name": "reverse",
+ "schema": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Boolean",
+ "type": "boolean"
+ }
+ },
+ {
+ "name": "max_size",
+ "schema": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "uint64",
+ "type": "integer",
+ "format": "uint64",
+ "minimum": 0.0
+ }
+ }
+ ],
+ "result": {
+ "name": "Vec < TransactionInfoView >",
+ "schema": {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Array_of_TransactionInfoView",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [
+ "block_hash",
+ "block_number",
+ "event_root_hash",
+ "gas_used",
+ "state_root_hash",
+ "status",
+ "transaction_hash",
+ "transaction_index"
+ ],
+ "properties": {
+ "block_hash": {
+ "type": "string",
+ "format": "HashValue"
+ },
+ "block_number": {
+ "type": "string"
+ },
+ "event_root_hash": {
+ "description": "The root hash of Merkle Accumulator storing all events emitted during this transaction.",
+ "type": "string",
+ "format": "HashValue"
+ },
+ "gas_used": {
+ "description": "The amount of gas used.",
+ "type": "string"
+ },
+ "state_root_hash": {
+ "description": "The root hash of Sparse Merkle Tree describing the world state at the end of this transaction.",
+ "type": "string",
+ "format": "HashValue"
+ },
+ "status": {
+ "description": "The vm status. If it is not `Executed`, this will provide the general error class. Execution failures and Move abort's receive more detailed information. But other errors are generally categorized with no status code or other information",
+ "anyOf": [
+ {
+ "type": "string",
+ "enum": [
+ "Executed",
+ "OutOfGas",
+ "MiscellaneousError"
+ ]
+ },
+ {
+ "type": "object",
+ "required": [
+ "MoveAbort"
+ ],
+ "properties": {
+ "MoveAbort": {
+ "type": "object",
+ "required": [
+ "abort_code",
+ "location"
+ ],
+ "properties": {
+ "abort_code": {
+ "type": "string"
+ },
+ "location": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "additionalProperties": false
+ },
+ {
+ "type": "object",
+ "required": [
+ "ExecutionFailure"
+ ],
+ "properties": {
+ "ExecutionFailure": {
+ "type": "object",
+ "required": [
+ "code_offset",
+ "function",
+ "location"
+ ],
+ "properties": {
+ "code_offset": {
+ "type": "integer",
+ "format": "uint16",
+ "minimum": 0.0
+ },
+ "function": {
+ "type": "integer",
+ "format": "uint16",
+ "minimum": 0.0
+ },
+ "location": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "additionalProperties": false
+ },
+ {
+ "type": "object",
+ "required": [
+ "Discard"
+ ],
+ "properties": {
+ "Discard": {
+ "type": "object",
+ "required": [
+ "status_code",
+ "status_code_name"
+ ],
+ "properties": {
+ "status_code": {
+ "type": "string"
+ },
+ "status_code_name": {
+ "type": "string"
+ }
+ }
+ }
+ },
+ "additionalProperties": false
+ }
+ ]
+ },
+ "transaction_hash": {
+ "description": "The hash of this transaction.",
+ "type": "string",
+ "format": "HashValue"
+ },
+ "transaction_index": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ }
+ }
+ }
+ }
+ }
}
]
}
\ No newline at end of file

diff:
@@ -23,6 +23,7 @@ use starcoin_types::block::{BlockInfo, BlockNumber};
use starcoin_types::filter::Filter;
use starcoin_types::startup_info::ChainInfo;
use starcoin_types::transaction::TransactionInfo;
+use std::collections::{HashMap, HashSet};
use std::convert::TryInto;
use std::sync::Arc;
pub struct ChainRpcImpl<S>
@@ -412,6 +413,54 @@ where
Box::pin(fut.boxed())
}
+
+ fn get_transaction_infos(
+ &self,
+ start_index: u64,
+ reverse: bool,
+ max_size: u64,
+ ) -> FutureResult<Vec<TransactionInfoView>> {
+ let service = self.service.clone();
+ let fut = async move {
+ let txn_infos = service
+ .get_txn_infos(start_index, reverse, max_size)
+ .await?;
+ // remove duplicate block_id
+ let id_sets = txn_infos
+ .iter()
+ .map(|info| info.block_id())
+ .collect::<HashSet<HashValue>>();
+ let block_ids = id_sets.into_iter().collect();
+ let blocks = service.get_blocks(block_ids).await?;
+ let mut block_cache = HashMap::new();
+ for block in blocks {
+ let block = block.ok_or_else(|| {
+ anyhow::anyhow!(
+ "cannot find block which start_index {}, reverse {}, max_size {}",
+ start_index,
+ reverse,
+ max_size
+ )
+ })?;
+
+ block_cache.insert(block.header.id(), block);
+ }
+
+ txn_infos
+ .into_iter()
+ .map(|info| {
+ let block_id = info.block_id();
+ TransactionInfoView::new(
+ Into::<(_, TransactionInfo)>::into(info).1,
+ block_cache.get(&block_id).unwrap(),
+ )
+ })
+ .collect::<Result<Vec<_>, _>>()
+ }
+ .map_err(map_err);
+
+ Box::pin(fut.boxed())
+ }
}
fn try_decode_block_txns(state: &dyn StateView, block: &mut BlockView) -> anyhow::Result<()> {

diff:
@@ -14,6 +14,7 @@ Feature: cmd integration test
Then cmd: "chain get-txn-infos @$.header.block_hash@"
Then cmd: "chain get-txn @$[0].transaction_hash@"
Then cmd: "chain get-events @$.transaction_hash@"
+ Then cmd: "chain get-txn-info-list -s 0 -c 5"
Then stop
Examples:

repo: starcoinorg/starcoin

original_message:
provider batch get transactioninfo (#2966)

* provider batch get transactioninfo

provider batch get transactioninfo

* provider batch get transactioninfo

provider batch get transactioninfo

* get batch transaction infos

get batch transaction infos

* provider get transaction infos

fmt code


message:
limit get_txn_info_list count size
* limit get_txn_info_list count size
limit get_txn_info_list count size
* limit get_txn_info_list size
limit get_txn_info_list size
* limit get_txn_info_list count size
limit get_txn_info_list count size

diff:
@@ -19,7 +19,7 @@ pub struct GetTransactionInfoListOpt {
#[structopt(name = "reverse", long, short = "r")]
reverse: Option<bool>,
- #[structopt(name = "count", long, short = "c")]
+ #[structopt(name = "count", long, short = "c", default_value = "32")]
count: u64,
}

diff:
@@ -25,6 +25,7 @@ const DEFAULT_WEB_SOCKET_PORT: u16 = 9870;
// UNSPECIFIED is 0.0.0.0
const DEFAULT_RPC_ADDRESS: IpAddr = IpAddr::V4(Ipv4Addr::UNSPECIFIED);
const DEFAULT_BLOCK_QUERY_MAX_RANGE: u64 = 32;
+const DEFAULT_TXN_INFO_QUEYR_MAX_RANGE: u64 = 32;
#[derive(Debug, Default, Clone, PartialEq, Deserialize, Serialize, StructOpt)]
pub struct HttpConfiguration {
@@ -386,6 +387,10 @@ pub struct RpcConfig {
#[serde(skip)]
#[structopt(skip)]
base: Option<Arc<BaseConfig>>,
+
+ #[serde(skip_serializing_if = "Option::is_none")]
+ #[structopt(long = "query-max-txn-info-range")]
+ pub txn_info_query_max_range: Option<u64>,
}
#[derive(Clone, Eq, PartialEq)]
@@ -451,6 +456,11 @@ impl RpcConfig {
.unwrap_or(DEFAULT_BLOCK_QUERY_MAX_RANGE)
}
+ pub fn txn_info_query_max_range(&self) -> u64 {
+ self.txn_info_query_max_range
+ .unwrap_or(DEFAULT_TXN_INFO_QUEYR_MAX_RANGE)
+ }
+
fn base(&self) -> &BaseConfig {
self.base.as_ref().expect("Config should init.")
}
@@ -523,6 +533,9 @@ impl ConfigModule for RpcConfig {
if opt.rpc.block_query_max_range.is_some() {
self.block_query_max_range = opt.rpc.block_query_max_range;
}
+ if opt.rpc.txn_info_query_max_range.is_some() {
+ self.txn_info_query_max_range = opt.rpc.txn_info_query_max_range;
+ }
self.http.merge(&opt.rpc.http)?;
self.tcp.merge(&opt.rpc.tcp)?;
self.ws.merge(&opt.rpc.ws)?;

diff:
@@ -421,9 +421,11 @@ where
max_size: u64,
) -> FutureResult<Vec<TransactionInfoView>> {
let service = self.service.clone();
+ let config = self.config.clone();
let fut = async move {
+ let max_return_num = max_size.min(config.rpc.txn_info_query_max_range());
let txn_infos = service
- .get_txn_infos(start_index, reverse, max_size)
+ .get_txn_infos(start_index, reverse, max_return_num)
.await?;
// remove duplicate block_id
let id_sets = txn_infos

original_message:
limit get_txn_info_list count size (#2975)

* limit get_txn_info_list count size

limit get_txn_info_list count size

* limit get_txn_info_list size

limit get_txn_info_list size

* limit get_txn_info_list count size

limit get_txn_info_list count size


message:
pprof with criterion for flamegraph
* pprof-rs with crition error
pprof-rs with crition error
* fix tmp path.
* add pprof with criterion for flamegraph
add pprof with criterion for flamegraph

diff:
@@ -329,6 +329,15 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
+[[package]]
+name = "arrayvec"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9"
+dependencies = [
+ "nodrop",
+]
+
[[package]]
name = "arrayvec"
version = "0.5.2"
@@ -667,6 +676,7 @@ dependencies = [
"futures-timer",
"network-api",
"parking_lot 0.11.2",
+ "pprof",
"proptest",
"rand 0.8.4",
"rand_core 0.6.3",
@@ -788,7 +798,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587"
dependencies = [
"arrayref",
- "arrayvec",
+ "arrayvec 0.5.2",
"constant_time_eq",
]
@@ -1021,6 +1031,12 @@ dependencies = [
"petgraph",
]
+[[package]]
+name = "bytemuck"
+version = "1.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72957246c41db82b8ef88a5486143830adeb8227ef9837740bdec67724cf2c5b"
+
[[package]]
name = "byteorder"
version = "0.5.3"
@@ -1458,6 +1474,15 @@ version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b"
+[[package]]
+name = "cpp_demangle"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ea47428dc9d2237f3c6bc134472edfd63ebba0af932e783506dcfd66f10d18a"
+dependencies = [
+ "cfg-if 1.0.0",
+]
+
[[package]]
name = "cpufeatures"
version = "0.2.1"
@@ -2006,6 +2031,15 @@ dependencies = [
"structopt 0.3.25",
]
+[[package]]
+name = "debugid"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f91cf5a8c2f2097e2a32627123508635d47ce10563d999ec1a95addf08b502ba"
+dependencies = [
+ "uuid",
+]
+
[[package]]
name = "derivative"
version = "2.2.0"
@@ -3650,6 +3684,24 @@ dependencies = [
"unindent",
]
+[[package]]
+name = "inferno"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa5bd9a10b38bf5f3c670f9d75c194adbecd2b1573f737668ab8599f41edc87"
+dependencies = [
+ "ahash 0.7.4",
+ "atty",
+ "indexmap",
+ "itoa",
+ "lazy_static",
+ "log 0.4.14",
+ "num-format",
+ "quick-xml",
+ "rgb",
+ "str_stack",
+]
+
[[package]]
name = "input_buffer"
version = "0.4.0"
@@ -4122,7 +4174,7 @@ version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374"
dependencies = [
- "arrayvec",
+ "arrayvec 0.5.2",
"bitflags",
"cfg-if 1.0.0",
"ryu",
@@ -4309,7 +4361,7 @@ version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50ed78489c87924235665a0ab345b298ee34dff0f7ad62c0ba6608b2144fb75e"
dependencies = [
- "arrayvec",
+ "arrayvec 0.5.2",
"asynchronous-codec 0.6.0",
"bytes 1.1.0",
"either",
@@ -5727,6 +5779,12 @@ dependencies = [
"hashbrown 0.8.2",
]
+[[package]]
+name = "nodrop"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
+
[[package]]
name = "nohash-hasher"
version = "0.2.0"
@@ -5806,6 +5864,16 @@ dependencies = [
"syn 1.0.80",
]
+[[package]]
+name = "num-format"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465"
+dependencies = [
+ "arrayvec 0.4.12",
+ "itoa",
+]
+
[[package]]
name = "num-integer"
version = "0.1.44"
@@ -6017,7 +6085,7 @@ version = "1.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4b26b16c7687c3075982af47719e481815df30bc544f7a6690763a25ca16e9d"
dependencies = [
- "arrayvec",
+ "arrayvec 0.5.2",
"bitvec 0.17.4",
"byte-slice-cast",
"serde",
@@ -6409,6 +6477,25 @@ dependencies = [
"winapi 0.2.8",
]
+[[package]]
+name = "pprof"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c7600124d694d855283caf9f333befe2abce090833bb638009aeddd9e156dee"
+dependencies = [
+ "backtrace",
+ "criterion",
+ "inferno",
+ "lazy_static",
+ "libc",
+ "log 0.4.14",
+ "nix 0.20.0",
+ "parking_lot 0.11.2",
+ "symbolic-demangle",
+ "tempfile",
+ "thiserror",
+]
+
[[package]]
name = "ppv-lite86"
version = "0.2.10"
@@ -6421,7 +6508,7 @@ version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad9940b913ee56ddd94aec2d3cd179dd47068236f42a1a6415ccf9d880ce2a61"
dependencies = [
- "arrayvec",
+ "arrayvec 0.5.2",
"typed-arena",
]
@@ -7242,6 +7329,15 @@ dependencies = [
"serde_json",
]
+[[package]]
+name = "rgb"
+version = "0.8.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fddb3b23626145d1776addfc307e1a1851f60ef6ca64f376bcb889697144cf0"
+dependencies = [
+ "bytemuck",
+]
+
[[package]]
name = "ring"
version = "0.16.20"
@@ -10028,6 +10124,12 @@ version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a66ae6a6cd930c97707cb3f1b62aadb8ddddd2fefa9df539564b3bfaa15dd31f"
+[[package]]
+name = "str_stack"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb"
+
[[package]]
name = "stream-task"
version = "1.8.0-alpha"
@@ -10136,6 +10238,29 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "171758edb47aa306a78dfa4ab9aeb5167405bd4e3dc2b64e88f6a84bbe98bd63"
+[[package]]
+name = "symbolic-common"
+version = "8.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1c77c64c4a8138a8d26b6ab26b29fdbe4fae7f6537cf9bf30e51b282efa2361"
+dependencies = [
+ "debugid",
+ "memmap",
+ "stable_deref_trait",
+ "uuid",
+]
+
+[[package]]
+name = "symbolic-demangle"
+version = "8.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "027d1ff6cc3cf63e523c60dfeaca5fb54c3ed15ecd9418f70a76854d966c79fb"
+dependencies = [
+ "cpp_demangle",
+ "rustc-demangle",
+ "symbolic-common",
+]
+
[[package]]
name = "syn"
version = "0.11.11"
@@ -11356,6 +11481,12 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372"
+[[package]]
+name = "uuid"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7"
+
[[package]]
name = "value-bag"
version = "1.0.0-alpha.6"

diff:
@@ -34,6 +34,7 @@ starcoin-network-api = {path = "../network/api", package="network-api"}
executor-benchmark = {path = "../executor/benchmark", package="starcoin-executor-benchmark" }
futures = "0.3.12"
logger = {path = "../commons/logger",package="starcoin-logger"}
+pprof = { version = "0.4", features = ["flamegraph", "criterion"] }
[dev-dependencies]
# see https://bheisler.github.io/criterion.rs/book/faq.html#cargo-bench-gives-unrecognized-option-errors-for-valid-command-line-options

diff:
@@ -11,3 +11,13 @@ cargo bench
```shell
cargo bench --bench bench_state_tree
```
+
+3. run a special benchmark with pprof
+```shell
+cargo bench --bench bench_state_tree -- --profile-time=10
+```
+Notice:
+run benchmark with pprof on macOS show this error, linux(ubuntu) run successful
+```shell
+error: process didn't exit successfully: `./bench_state_tree-3daab626ee8d6834 --profile-time=10 --bench` (signal: 11, SIGSEGV: invalid memory reference)
+```
\ No newline at end of file

diff:
use benchmarks::chain::ChainBencher;
#[allow(deprecated)]
use criterion::{criterion_group, criterion_main, Benchmark, Criterion};
+use pprof::criterion::{Output, PProfProfiler};
#[allow(deprecated)]
fn block_apply(c: &mut Criterion) {
@@ -38,5 +39,9 @@ fn query_block(c: &mut Criterion) {
}
}
-criterion_group!(starcoin_chain_benches, block_apply, query_block);
+criterion_group!(
+ name=starcoin_chain_benches;
+ config = Criterion::default()
+ .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+ targets=block_apply,query_block);
criterion_main!(starcoin_chain_benches);

diff:
@@ -5,6 +5,7 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
use crypto::hash::*;
use forkable_jellyfish_merkle::blob::Blob;
use forkable_jellyfish_merkle::HashValueKey;
+use pprof::criterion::{Output, PProfProfiler};
use rand::{rngs::StdRng, SeedableRng};
use starcoin_config::RocksdbConfig;
use starcoin_state_store_api::StateNodeStore;
@@ -98,7 +99,11 @@ fn bench_put_and_commit(c: &mut Criterion) {
group.finish();
}
-criterion_group!(benches, bench_get_with_proof, bench_put_and_commit);
+criterion_group!(
+ name=benches;
+ config = Criterion::default()
+ .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+ targets=bench_get_with_proof, bench_put_and_commit);
criterion_main!(benches);
fn gen_kv_from_seed(seed: &[u8], num_keys: usize) -> HashMap<HashValueKey, Blob> {

diff:
use benchmarks::storage::StorageBencher;
use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
use crypto::HashValue;
+use pprof::criterion::{Output, PProfProfiler};
use starcoin_accumulator::{accumulator_info::AccumulatorInfo, Accumulator, MerkleAccumulator};
use starcoin_config::RocksdbConfig;
use starcoin_storage::cache_storage::CacheStorage;
@@ -16,15 +17,11 @@ use std::sync::Arc;
//
fn storage_transaction(c: &mut Criterion) {
::logger::init_for_test();
+ let path = starcoin_config::temp_path();
c.bench_function("storage_transaction", |b| {
let storage = Storage::new(StorageInstance::new_cache_and_db_instance(
CacheStorage::new(None),
- DBStorage::new(
- starcoin_config::temp_path().as_ref(),
- RocksdbConfig::default(),
- None,
- )
- .unwrap(),
+ DBStorage::new(path.as_ref(), RocksdbConfig::default(), None).unwrap(),
))
.unwrap();
let bencher = StorageBencher::new(storage);
@@ -35,16 +32,12 @@ fn storage_transaction(c: &mut Criterion) {
/// accumulator benchmarks
fn accumulator_append(c: &mut Criterion) {
::logger::init_for_test();
+ let path = starcoin_config::temp_path();
c.bench_function("accumulator_append", |b| {
let storage = Arc::new(
Storage::new(StorageInstance::new_cache_and_db_instance(
CacheStorage::new(None),
- DBStorage::new(
- starcoin_config::temp_path().as_ref(),
- RocksdbConfig::default(),
- None,
- )
- .unwrap(),
+ DBStorage::new(path.as_ref(), RocksdbConfig::default(), None).unwrap(),
))
.unwrap(),
);
@@ -70,8 +63,9 @@ fn create_leaves(nums: std::ops::Range<usize>) -> Vec<HashValue> {
.collect()
}
criterion_group!(
- starcoin_storage_benches,
- storage_transaction,
- accumulator_append
+ name=starcoin_storage_benches;
+ config = Criterion::default()
+ .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+ targets=storage_transaction, accumulator_append
);
criterion_main!(starcoin_storage_benches);

diff:
// SPDX-License-Identifier: Apache-2.0
use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
use executor_benchmark::run_benchmark;
+use pprof::criterion::{Output, PProfProfiler};
pub fn transaction_execution(c: &mut Criterion) {
::logger::init();
@@ -15,5 +16,9 @@ pub fn transaction_execution(c: &mut Criterion) {
}
}
-criterion_group!(starcoin_vm_benches, transaction_execution);
+criterion_group!(
+ name=starcoin_vm_benches;
+ config = Criterion::default()
+ .with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+ targets=transaction_execution);
criterion_main!(starcoin_vm_benches);

original_message:
pprof with criterion for flamegraph (#2992)

* pprof-rs with crition error

pprof-rs with crition error

* fix tmp path.

* add pprof with criterion for flamegraph

add pprof with criterion for flamegraph

Co-authored-by: jolestar <jolestar@gmail.com>


message:
pprof use only linux
pprof use only linux

diff:
@@ -34,6 +34,7 @@ starcoin-network-api = {path = "../network/api", package="network-api"}
executor-benchmark = {path = "../executor/benchmark", package="starcoin-executor-benchmark" }
futures = "0.3.12"
logger = {path = "../commons/logger",package="starcoin-logger"}
+[target."cfg(target_os=\"linux\")".dependencies]
pprof = { version = "0.4", features = ["flamegraph", "criterion"] }
[dev-dependencies]

diff:
@@ -12,7 +12,7 @@ cargo bench
cargo bench --bench bench_state_tree
```
-3. run a special benchmark with pprof
+3. run a special benchmark with pprof (on linux)
```shell
cargo bench --bench bench_state_tree -- --profile-time=10
```

diff:
use benchmarks::chain::ChainBencher;
#[allow(deprecated)]
use criterion::{criterion_group, criterion_main, Benchmark, Criterion};
+#[cfg(target_os = "linux")]
use pprof::criterion::{Output, PProfProfiler};
#[allow(deprecated)]
@@ -38,10 +39,12 @@ fn query_block(c: &mut Criterion) {
}
}
}
-
+#[cfg(target_os = "linux")]
criterion_group!(
name=starcoin_chain_benches;
config = Criterion::default()
.with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
targets=block_apply,query_block);
+#[cfg(not(target_os = "linux"))]
+criterion_group!(starcoin_chain_benches, block_apply, query_block);
criterion_main!(starcoin_chain_benches);

diff:
@@ -5,6 +5,7 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
use crypto::hash::*;
use forkable_jellyfish_merkle::blob::Blob;
use forkable_jellyfish_merkle::HashValueKey;
+#[cfg(target_os = "linux")]
use pprof::criterion::{Output, PProfProfiler};
use rand::{rngs::StdRng, SeedableRng};
use starcoin_config::RocksdbConfig;
@@ -98,12 +99,14 @@ fn bench_put_and_commit(c: &mut Criterion) {
group.finish();
}
-
+#[cfg(target_os = "linux")]
criterion_group!(
name=benches;
config = Criterion::default()
.with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
targets=bench_get_with_proof, bench_put_and_commit);
+#[cfg(not(target_os = "linux"))]
+criterion_group!(benches, bench_get_with_proof, bench_put_and_commit);
criterion_main!(benches);
fn gen_kv_from_seed(seed: &[u8], num_keys: usize) -> HashMap<HashValueKey, Blob> {

diff:
use benchmarks::storage::StorageBencher;
use criterion::{criterion_group, criterion_main, BatchSize, Criterion};
use crypto::HashValue;
+#[cfg(target_os = "linux")]
use pprof::criterion::{Output, PProfProfiler};
use starcoin_accumulator::{accumulator_info::AccumulatorInfo, Accumulator, MerkleAccumulator};
use starcoin_config::RocksdbConfig;
@@ -62,10 +63,17 @@ fn create_leaves(nums: std::ops::Range<usize>) -> Vec<HashValue> {
nums.map(|x| HashValue::sha3_256_of(x.to_be_bytes().as_ref()))
.collect()
}
+#[cfg(target_os = "linux")]
criterion_group!(
name=starcoin_storage_benches;
config = Criterion::default()
.with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
targets=storage_transaction, accumulator_append
);
+#[cfg(not(target_os = "linux"))]
+criterion_group!(
+ starcoin_storage_benches,
+ storage_transaction,
+ accumulator_append
+);
criterion_main!(starcoin_storage_benches);

diff:
// SPDX-License-Identifier: Apache-2.0
use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
use executor_benchmark::run_benchmark;
+#[cfg(target_os = "linux")]
use pprof::criterion::{Output, PProfProfiler};
pub fn transaction_execution(c: &mut Criterion) {
@@ -16,9 +17,12 @@ pub fn transaction_execution(c: &mut Criterion) {
}
}
+#[cfg(target_os = "linux")]
criterion_group!(
name=starcoin_vm_benches;
config = Criterion::default()
.with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
targets=transaction_execution);
+#[cfg(not(target_os = "linux"))]
+criterion_group!(starcoin_vm_benches, transaction_execution);
criterion_main!(starcoin_vm_benches);

original_message:
pprof use only linux (#3004)

pprof use only linux


message:
unsave block body
unsave block body

diff:
@@ -180,13 +180,12 @@ impl BlockStorage {
}
pub fn commit_block(&self, block: Block) -> Result<()> {
- let (header, body) = block.clone().into_inner();
+ let (header, _) = block.clone().into_inner();
//save header
- let block_id = header.id();
self.save_header(header)?;
- //save body
- self.save_body(block_id, body)?;
- //save block cache
+ // save block , no need body
+ // self.save_body(block_id, body)?;
+ //save block
self.save(block)
}
pub fn delete_block(&self, block_id: HashValue) -> Result<()> {

diff:
@@ -295,6 +295,7 @@ impl BlockStore for Storage {
self.block_storage.get_blocks(ids)
}
+ #[allow(deprecated)]
fn get_body(&self, block_id: HashValue) -> Result<Option<BlockBody>> {
self.block_storage.get_body(block_id)
}

original_message:
unsave block body (#2995)

unsave block body