message: Add save action warnings

diff:
}
.create-modal .submit {
- width: 100%;
- margin: 24px 32px 32px;
+ width: auto;
+ margin: 24px 32px 16px;
+}
+
+.create-modal .save-warning {
+ font-size: 12px;
+ margin: 0 32px 32px;
+ opacity: 0.7;
}
.create-modal .row {

diff:
@@ -10,7 +10,12 @@ import RequestError from './request-error.jsx';
import TimedMessage from './timed-message.jsx';
import CronPicker from '../cron-picker/cron-picker.jsx';
-import {getHighlightMode, DEFAULT_REFRESH_INTERVAL, WAITING_MESSAGE} from '../../../constants/constants.js';
+import {
+ getHighlightMode,
+ DEFAULT_REFRESH_INTERVAL,
+ WAITING_MESSAGE,
+ SAVE_WARNING
+} from '../../../constants/constants.js';
import './create-modal.css';
@@ -159,7 +164,7 @@ class CreateModal extends Component {
})}
>

- Frequency
+ Interval

<CronPicker onChange={this.handleIntervalChange} />
@@ -182,9 +187,12 @@ class CreateModal extends Component {
{this.state.successMessage ? (
<SuccessMessage>{this.state.successMessage}</SuccessMessage>
) : (
+ <Column>
<button type="submit" className="submit" onClick={this.submit}>
{this.state.saving ? 'Saving...' : 'Schedule Query'}
</button>
+
{SAVE_WARNING}

+ </Column>
)}
</Row>
</Column>

diff:
-import React, {Component} from 'react';
+import React, {Component, Fragment} from 'react';
import PropTypes from 'prop-types';
import {Controlled as CodeMirror} from 'react-codemirror2';
import ms from 'ms';
@@ -13,7 +13,12 @@ import CronPicker from '../cron-picker/cron-picker.jsx';
import {Row, Column} from '../../layout.jsx';
import SQL from './sql.jsx';
import {plotlyUrl} from '../../../utils/utils.js';
-import {getHighlightMode, DEFAULT_REFRESH_INTERVAL, WAITING_MESSAGE} from '../../../constants/constants.js';
+import {
+ getHighlightMode,
+ DEFAULT_REFRESH_INTERVAL,
+ WAITING_MESSAGE,
+ SAVE_WARNING
+} from '../../../constants/constants.js';
import {getInitialCronMode} from '../cron-picker/cron-helpers.js';
const NO_OP = () => {};
@@ -59,6 +64,7 @@ export class PreviewModal extends Component {
this.onSubmit = this.onSubmit.bind(this);
this.onDelete = this.onDelete.bind(this);
this.close = this.close.bind(this);
+ this.renderButtonRow = this.renderButtonRow.bind(this);
this.handleIntervalChange = this.handleIntervalChange.bind(this);
}
@@ -115,6 +121,53 @@ export class PreviewModal extends Component {
this.props.onClickAway();
}
+ renderButtonRow() {
+ const {loading, editing} = this.state;
+ const canEdit = this.props.currentRequestor && this.props.currentRequestor === this.props.query.requestor;
+ const success = this.state.successMessage;
+
+ if (!canEdit) {
+ return (
+ <button style={{margin: 0}} onClick={this.props.onLogin}>
+ Log in to edit query
+ </button>
+ );
+ }
+
+ if (success) {
+ return <SuccessMessage>{this.state.successMessage}</SuccessMessage>;
+ }
+
+ if (editing) {
+ return (
+ <Column>
+ <button style={{margin: 0}} onClick={this.onSubmit}>
+ {loading ? 'Saving...' : 'Save'}
+ </button>
+
{SAVE_WARNING}

+ </Column>
+ );
+ }
+
+ return (
+ <Fragment>
+ <button style={{margin: 0}} onClick={this.onSubmit}>
+ Edit
+ </button>
+ <button
+ style={{
+ margin: 0,
+ border: 'none',
+ background: 'red'
+ }}
+ onClick={this.onDelete}
+ >
+ {this.state.confirmedDelete ? 'Click to confirm' : 'Delete'}
+ </button>
+ </Fragment>
+ );
+ }
+
render() {
const props = this.props;
@@ -128,8 +181,6 @@ export class PreviewModal extends Component {
const initialModeId = getInitialCronMode(props.query);
- const canEdit = this.props.currentRequestor && this.props.currentRequestor === props.query.requestor;
- const success = this.state.successMessage;
content = (
<Column style={{width: '60%', minWidth: 640, background: 'white'}}>
<Row
@@ -179,7 +230,7 @@ export class PreviewModal extends Component {

</Row>
<Row style={rowStyle}>
-
Frequency

+
Interval

{editing ? (

<CronPicker onChange={this.handleIntervalChange} initialModeId={initialModeId} />
@@ -228,33 +279,7 @@ export class PreviewModal extends Component {
paddingBottom: 0
}}
>
- {!success &&
- !canEdit && (
- <button style={{margin: 0}} onClick={this.props.onLogin}>
- Log in to edit query
- </button>
- )}
- {!success &&
- canEdit && (
- <button style={{margin: 0}} onClick={this.onSubmit}>
- {loading ? 'Saving...' : editing ? 'Save' : 'Edit'}
- </button>
- )}
- {!success &&
- canEdit &&
- !editing && (
- <button
- style={{
- margin: 0,
- border: 'none',
- background: 'red'
- }}
- onClick={this.onDelete}
- >
- {this.state.confirmedDelete ? 'Click to confirm' : 'Delete'}
- </button>
- )}
- {success && <SuccessMessage>{this.state.successMessage}</SuccessMessage>}
+ {this.renderButtonRow()}
</Row>
</Column>
</Column>

diff:
@@ -512,3 +512,6 @@ export function getHighlightMode(dialect) {
export const WAITING_MESSAGE = 'This may take a long time. Your query is ' +
'currently executing and must finish before it can be saved.';
+
+export const SAVE_WARNING = 'Note: when you click the button below, the query will ' +
+ 'execute and update the dataset immediately. Thereafter it will do so at the requested interval.';

original_message: Add save action warnings


message: Wrap lines on CreateModal

change_type: MODIFY

diff:
@@ -70,6 +70,7 @@ class CreateModal extends Component {
saving: false
};
this.options = {
+ lineWrapping: true,
lineNumbers: true,
tabSize: 4,
readOnly: false,

license: MIT License

repo: plotly/falcon

original_message: Wrap lines on CreateModal


message: Fix overflow on Schedule and Query pages

diff:
@@ -22,8 +22,7 @@ a:hover {
body {
background-color: #F3F6FA;
position: relative;
- height: 100vh;
- overflow-y: scroll;
+ overflow-y: auto;
font-weight: 200;
font-size: 1.5em; /* currently ems cause chrome bug misinterpreting rems on body element */
line-height: 1.6;


message: 'Interval' -> 'Schedule' in scheduled table

diff:
@@ -141,7 +141,7 @@ class Scheduler extends Component {
},
{
key: 'run',
- name: 'Interval',
+ name: 'Schedule',
filterable: true,
formatter: IntervalFormatter
}


message: Add text-overflow for long interval strings

diff:
@@ -69,10 +69,10 @@ class IntervalFormatter extends React.Component {
const run = this.props.value;
return (
- <Row>
- <Column>
<em
+ className="ellipsis"
style={{
+ display: 'block',
fontSize: 15
}}
>
@@ -82,8 +82,6 @@ class IntervalFormatter extends React.Component {
long: true
})}`}
</em>
- </Column>
- </Row>
);
}
}


message: Misc modal style fixes

diff:
width: 80%;
}
-.create-modal .row-body pre {
+.create-modal .row-body > pre {
margin-top: 5px;
+ overflow-y: auto;
+ max-height: 300px;
}

diff:
@@ -123,7 +123,7 @@ class CreateModal extends Component {
render() {
return (
<Modal open={this.props.open} onClickAway={this.props.onClickAway} className="scheduler create-modal">
- <Column className="container" style={{width: '60%', maxHeight: '100vh', minWidth: 640}}>
+ <Column className="container" style={{width: '60%', maxHeight: '100vh', minWidth: 640, paddingBottom: '16px'}}>
<Row>
<Column className="innerColumn">

Create Scheduled Query

diff:
@@ -207,11 +207,12 @@ export class PreviewModal extends Component {
const initialModeId = getInitialCronMode(props.query);
content = (
- <Column style={{width: '60%', maxHeight: '100vh', minWidth: 640, background: 'white'}}>
+ <Column style={{width: '60%', maxHeight: '100vh', minWidth: 640, background: 'white', paddingTop: 16}}>
+ { editing && <Row style={{ padding: '0 32px', justifyContent: 'flex-start', fontSize: 12, marginTop: 8, fontWeight: 600, opacity: 0.4, letterSpacing: '0.5px' }}>EDITING</Row> }
<Row
className="sql-preview"
style={{
- padding: '16px 32px',
+ padding: '0 32px 16px',
position: 'relative',
justifyContent: 'flex-start'
}}
@@ -234,9 +235,9 @@ export class PreviewModal extends Component {
<Column style={{background: '#F5F7FB', padding: '16px 32px'}}>
<Row style={rowStyle}>

Query

-

+

{editing ? (
-

+

<CodeMirror
options={{
lineNumbers: true,
@@ -323,7 +324,7 @@ export class PreviewModal extends Component {
justifyContent: 'space-between',
border: 'none',
marginTop: 48,
- paddingBottom: 0
+ paddingBottom: 16
}}
>
{this.renderButtonRow()}

diff:
}
.meta-preview button {
- min-height: 28px;
+ min-height: 21px;
}

original_message: Misc modal style fixes


message: Fix scheduler double border

diff:
}
.scheduler-table .react-grid-Row {
- border-top: 1px solid #c8d4e3;
cursor: pointer;
}
-.scheduler-table .react-grid-Row:first-child {
- border-top: none;
-}
-
.sql pre {
margin: 0 auto;
max-height: 100%;

original_message: Fix scheduler double border


message: Add query name validation

diff:
@@ -104,6 +104,9 @@ class CreateModal extends Component {
if (!this.state.interval) {
return this.setState({error: 'Please select an interval above.'});
}
+ if (this.state.name && this.state.name.trim().length === 0) {
+ return this.setState({error: 'Please enter a valid query name above.'});
+ }
this.setState({saving: true, error: null});
this.props

diff:
@@ -87,6 +87,11 @@ export class PreviewModal extends Component {
if (this.state.editing) {
const {connectionId, fid, requestor, uids, refreshInterval} = this.props.query;
const {code: query, cronInterval, name} = this.state;
+
+ if (this.state.name && this.state.name.trim().length === 0) {
+ return this.setState({error: 'Please enter a valid query name above.'});
+ }
+
this.setState({loading: true, error: null});
this.props
.onSave({

original_message: Add query name validation


message: Copy query to clipboard

diff:
@@ -426,6 +426,7 @@ class Settings extends Component {
openLogin={this.openLogin}
requestor={username}
dialect={dialect}
+ preview={preview}
openQueryPage={() => this.updateSelectedPanel(1)}
/>
</TabPanel>

diff:
padding: 16px;
}
-.login-modal .submit {
+.login-modal .actions button {
width: 50%;
margin: 24px 32px 32px;
}

diff:
import React from 'react';
import PropTypes from 'prop-types';
+import {CopyToClipboard} from 'react-copy-to-clipboard';
import {Row, Column} from '../../layout.jsx';
import Modal from '../../modal.jsx';
@@ -16,14 +17,19 @@ const PromptLoginModal = props => (
</button>
<Row className="header">

- To create a scheduled query, you'll need to be logged into Plotly. Logging in will reset your query,
- so please save it elsewhere before doing so.
+ To create a scheduled query, you'll need to be logged into Plotly.

+ Note: logging in will reset your query, click the button below to copy the query to your clipboard.

</Row>
- <Row>
- <button type="submit" className="submit" onClick={props.onSubmit}>
+ <Row className="actions">
+ <button type="submit" onClick={props.onSubmit}>
Log In
</button>
+ <CopyToClipboard text={props.preview.code}>
+ <button className="btn-secondary">
+ Copy Query
+ </button>
+ </CopyToClipboard>
</Row>
</Column>
</Modal>
@@ -31,6 +37,7 @@ const PromptLoginModal = props => (
PromptLoginModal.propTypes = {
open: PropTypes.bool.isRequired,
+ preview: PropTypes.string.isRequired,
onClickAway: PropTypes.func.isRequired,
onSubmit: PropTypes.func.isRequired
};

diff:
@@ -115,6 +115,7 @@ class Scheduler extends Component {
initialCode: PropTypes.string,
requestor: PropTypes.string,
dialect: PropTypes.string,
+ preview: PropTypes.string,
refreshQueries: PropTypes.func.isRequired,
openLogin: PropTypes.func.isRequired,
createScheduledQuery: PropTypes.func.isRequired,
@@ -304,6 +305,7 @@ class Scheduler extends Component {
open={!loggedIn && this.state.createModalOpen}
onClickAway={this.closeCreateModal}
onSubmit={this.props.openLogin}
+ preview={this.props.preview}
/>
{/*

diff:
@@ -2428,6 +2428,12 @@ copy-descriptor@^0.1.0:
version "0.1.1"
resolved "https://registry.yarnpkg.com/copy-descriptor/-/copy-descriptor-0.1.1.tgz#676f6eb3c39997c2ee1ac3a924fd6124748f578d"
+copy-to-clipboard@^3:
+ version "3.0.8"
+ resolved "https://registry.yarnpkg.com/copy-to-clipboard/-/copy-to-clipboard-3.0.8.tgz#f4e82f4a8830dce4666b7eb8ded0c9bcc313aba9"
+ dependencies:
+ toggle-selection "^1.0.3"
+
core-js@^1.0.0:
version "1.2.7"
resolved "https://registry.yarnpkg.com/core-js/-/core-js-1.2.7.tgz#652294c14651db28fa93bd2d5ff2983a4f08c636"
@@ -8644,6 +8650,13 @@ react-cookies@^0.1.0:
cookie "^0.3.1"
object-assign "^4.1.1"
+react-copy-to-clipboard@^5.0.1:
+ version "5.0.1"
+ resolved "https://registry.yarnpkg.com/react-copy-to-clipboard/-/react-copy-to-clipboard-5.0.1.tgz#8eae107bb400be73132ed3b6a7b4fb156090208e"
+ dependencies:
+ copy-to-clipboard "^3"
+ prop-types "^15.5.8"
+
react-data-grid-addons@^3.0.11:
version "3.0.11"
resolved "https://registry.yarnpkg.com/react-data-grid-addons/-/react-data-grid-addons-3.0.11.tgz#9d38c31aa1f475f93735246292395e6d1fa7eb38"
@@ -10764,6 +10777,10 @@ to-regex@^3.0.1, to-regex@^3.0.2:
regex-not "^1.0.2"
safe-regex "^1.1.0"
+toggle-selection@^1.0.3:
+ version "1.0.6"
+ resolved "https://registry.yarnpkg.com/toggle-selection/-/toggle-selection-1.0.6.tgz#6e45b1263f2017fa0acc7d89d78b15b8bf77da32"
+
tohash@^1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/tohash/-/tohash-1.0.2.tgz#9e66e497da0cfd77ba85f9663065adf2d8c99981"

original_message: Copy query to clipboard


message: Fix close button position, lint

diff:
@@ -17,7 +17,8 @@ const PromptLoginModal = props => (
</button>
<Row className="header">

- To create a scheduled query, you'll need to be logged into Plotly.

+ To create a scheduled query, you'll need to be logged into Plotly.

+

Note: logging in will reset your query, click the button below to copy the query to your clipboard.

</Row>
@@ -26,9 +27,7 @@ const PromptLoginModal = props => (
Log In
</button>
<CopyToClipboard text={props.preview.code}>
- <button className="btn-secondary">
- Copy Query
- </button>
+ <button className="btn-secondary">Copy Query</button>
</CopyToClipboard>
</Row>
</Column>

diff:
@@ -212,7 +212,28 @@ export class PreviewModal extends Component {
const initialModeId = getInitialCronMode(props.query);
content = (
- <Column style={{width: '60%', maxHeight: '100vh', minWidth: 640, background: 'white', paddingTop: 16}}>
+ <Column
+ style={{
+ width: '60%',
+ maxHeight: '100vh',
+ minWidth: 640,
+ background: 'white',
+ paddingTop: 16,
+ position: 'relative'
+ }}
+ >
+ <button
+ onClick={this.close}
+ style={{
+ position: 'absolute',
+ top: '16px',
+ right: '16px',
+ padding: '2px 4px',
+ zIndex: 99
+ }}
+ >
+ ×
+ </button>
{editing && (
<Row
style={{
@@ -239,17 +260,6 @@ export class PreviewModal extends Component {

{this.state.name ? {this.state.name} : <SQL className="bold">{this.state.code}</SQL>}

- <button
- onClick={this.close}
- style={{
- position: 'absolute',
- top: '16px',
- right: '16px',
- padding: '2px 4px'
- }}
- >
- ×
- </button>
</Row>
<Column style={{background: '#F5F7FB', padding: '16px 32px'}}>
<Row style={rowStyle}>

original_message: Fix close button position, lint


message: clean up SuccessMessage, show datasetUrl in create-modal

diff:
@@ -6,12 +6,14 @@ import cronstrue from 'cronstrue';
import {Row, Column} from '../../layout.jsx';
import Modal from '../../modal.jsx';
+import {Link} from '../../Link.react';
import SuccessMessage from '../../success.jsx';
import RequestError from './request-error.jsx';
import TimedMessage from './timed-message.jsx';
import CronPicker from '../cron-picker/cron-picker.jsx';
import SQL from './sql.jsx';
+import {datasetUrl as getDatasetURL} from '../../../utils/utils';
import {getHighlightMode, WAITING_MESSAGE, SAVE_WARNING} from '../../../constants/constants.js';
import './create-modal.css';
@@ -62,7 +64,8 @@ class CreateModal extends Component {
name: props.initialName,
interval: '*/5 * * * *',
error: null,
- saving: false
+ saving: false,
+ datasetUrl: null
};
this.options = {
lineWrapping: true,
@@ -108,8 +111,12 @@ class CreateModal extends Component {
cronInterval: this.state.interval,
name: this.state.name ? this.state.name.trim() : ''
})
- .then(() => {
- this.setState({successMessage: 'Scheduled query saved successfully!', saving: false});
+ .then((res) => {
+ this.setState({
+ successMessage: 'Scheduled query saved successfully!',
+ saving: false,
+ datasetUrl: getDatasetURL(res.fid)
+ });
})
.catch(error => this.setState({error: error.message, saving: false}));
}
@@ -193,7 +200,11 @@ class CreateModal extends Component {
</Column>
<Row>
{this.state.successMessage ? (
- <SuccessMessage>{this.state.successMessage}</SuccessMessage>
+ <Column style={{ padding: '0 32px 16px' }}>
+ <SuccessMessage message={this.state.successMessage}>
+ <Link href={this.state.datasetUrl}>View Live Dataset</Link>
+ </SuccessMessage>
+ </Column>
) : (
<Column>
<button type="submit" className="submit" onClick={this.submit}>

diff:
@@ -12,7 +12,7 @@ import {Link} from '../../Link.react.js';
import CronPicker from '../cron-picker/cron-picker.jsx';
import {Row, Column} from '../../layout.jsx';
import SQL from './sql.jsx';
-import {plotlyUrl} from '../../../utils/utils.js';
+import {datasetUrl} from '../../../utils/utils.js';
import {getHighlightMode, WAITING_MESSAGE, SAVE_WARNING} from '../../../constants/constants.js';
import {getInitialCronMode} from '../cron-picker/cron-helpers.js';
@@ -156,7 +156,11 @@ export class PreviewModal extends Component {
}
if (success) {
- return <SuccessMessage>{this.state.successMessage}</SuccessMessage>;
+ return (
+ <Column>
+ <SuccessMessage>{this.state.successMessage}</SuccessMessage>
+ </Column>
+ );
}
if (editing) {
@@ -196,8 +200,7 @@ export class PreviewModal extends Component {
if (!props.query) {
content = null;
} else {
- const [account, gridId] = props.query.fid.split(':');
- const link = `${plotlyUrl()}/~${account}/${gridId}`;
+ const link = datasetUrl(props.query.fid);
const {editing, loading} = this.state;
const initialModeId = getInitialCronMode(props.query);

diff:
import React from 'react';
import PropTypes from 'prop-types';
-const style = {marginBottom: 16, fontSize: 16, color: '#00cc96'};
+const style = {fontSize: 16};
const SuccessMessage = (props) => (
-
+

+
{props.message}

{props.children}
-
+

);
SuccessMessage.propTypes = {
- children: PropTypes.node
+ children: PropTypes.node,
+ message: PropTypes.string
};
export default SuccessMessage;

diff:
@@ -59,6 +59,11 @@ export function homeUrl() {
'';
}
+export function datasetUrl(fid) {
+ const [account, gridId] = fid.split(':');
+ return `${plotlyUrl()}/~${account}/${gridId}`;
+}
+
export function getPathNames(url) {
const parser = document.createElement('a');
parser.href = url;


message: fix scheduler auto height fitting

diff:
min-height: calc(100vh - 280px) !important;
}
+.scheduler-table .react-grid-Canvas {
+ height: calc(100vh - 312px) !important;
+}
+
.scheduler-table pre {
margin: 0;
}

original_message: fix scheduler auto height fitting


message: add drop down connection select

diff:
import React from 'react';
import PropTypes from 'prop-types';
-import {values} from 'ramda';
+import Select from 'react-select';
import {
DIALECTS, LOGOS
} from '../../../constants/constants';
-import classnames from 'classnames';
+
+class DialectOption extends React.Component {
+ static propTypes = {
+ children: PropTypes.node,
+ className: PropTypes.string,
+ isDisabled: PropTypes.bool,
+ isFocused: PropTypes.bool,
+ isSelected: PropTypes.bool,
+ onFocus: PropTypes.func,
+ onSelect: PropTypes.func,
+ option: PropTypes.object.isRequired
+ };
+
+ constructor(props) {
+ super(props);
+
+ this.handleMouseDown = this.handleMouseDown.bind(this);
+ this.handleMouseEnter = this.handleMouseEnter.bind(this);
+ this.handleMouseMove = this.handleMouseMove.bind(this);
+ }
+
+ handleMouseDown(event) {
+ event.preventDefault();
+ event.stopPropagation();
+ this.props.onSelect(this.props.option, event);
+ }
+
+ handleMouseEnter(event) {
+ this.props.onFocus(this.props.option, event);
+ }
+
+ handleMouseMove(event) {
+ if (this.props.isFocused) return;
+ this.props.onFocus(this.props.option, event);
+ }
+
+ render() {
+ const logoStyle = {
+ display: 'inline-block',
+ height: '100%',
+ width: '100%',
+ objectFit: 'contain',
+ position: 'relative'
+ };
+
+ const logoContainerStyle = {height: '100%', width: 64, marginRight: 16};
+
+ const optionStyle = {
+ height: 56,
+ display: 'flex',
+ justifyContent: 'flex-start',
+ alignItems: 'center'
+ };
+
+ return (
+

+

+
+

+ {this.props.children}
+

+ );
+ }
+}
+
+class DialectValue extends React.Component {
+ static propTypes = {
+ children: PropTypes.node,
+ value: PropTypes.object
+ };
+
+ render () {
+ const logoStyle = {
+ display: 'inline-block',
+ marginRight: 10,
+ position: 'relative',
+ verticalAlign: 'middle'
+ };
+
+ return (
+

+
+
+ {this.props.children}
+
+

+ );
+ }
+}
/**
* The following is the Dialect Selector
@@ -17,30 +112,36 @@ import classnames from 'classnames';
const DialectSelector = function DialectSelector(props) {
const {connectionObject, updateConnection} = props;
- const logos = values(DIALECTS).map(DIALECT => (
-
 {
- updateConnection(
- {dialect: DIALECT}
- );
- }}
- id={`test-logo-${DIALECT}`}
- >
- <img
- className={'logoImage'}
- src={LOGOS[DIALECT]}
+ // react-select requires options to have `label` and `value` keys
+ const DIALECT_OPTIONS = Object.entries(DIALECTS).map(DIALECT => ({
+ label: DIALECT[1],
+ value: DIALECT[0]
+ }));
+
+ const placeholder = Select a database type...;
+
+ const onChange = ({ label }) => updateConnection({dialect: label});
+
+ const currentValue = DIALECT_OPTIONS.filter(DIALECT =>
+ DIALECT.label === connectionObject.dialect)[0];
+
+ return (
+

+ Database Type
+

+ <Select
+ searchable={false}
+ clearable={false}
+ onChange={onChange}
+ optionComponent={DialectOption}
+ options={DIALECT_OPTIONS}
+ placeholder={placeholder}
+ value={currentValue}
+ valueComponent={DialectValue}
/>

- ));
-
- return
{logos}
;
+

+ );
};
DialectSelector.propTypes = {

diff:
@@ -122,7 +122,6 @@ class Settings extends Component {
)}
>
<div className={'dialectSelector'}>
- <ReactToolTip place={'top'} type={'dark'} effect={'solid'} />
<DialectSelector
connectionObject={connectionObject}
updateConnection={updateConnection}

diff:
-.logoContainer {
- margin: 20px;
- width: 800px;
- text-align: center;
-}
-
-.logo {
- display: inline-block;
- width: 100px;
- height: 70px;
- padding: 2px;
- margin: 5px;
- border-color: #BEC8D9;
- border-width: 2px;
- border-style: solid;
- border-radius: 5px;
- vertical-align: middle;
-}
-
.logoImage {
height: auto;
- max-height: 100%;
+ max-height: 75%;
max-width: 100%;
width: auto;
}
-
-.logo:hover, .logoSelected {
- border-color: #8BB5FE;
- cursor: pointer;
- border-width: 4px;
- margin: 3px;
-}
-
-.dialectInput {
- display: inline-block;
-}
-
-.dialectLabel {
- display: inline-block;
-}

diff:
}
.dialectSelector {
- display: inline-block;
+ margin-top: 32px;
+}
+
+.dialectSelector .Select {
+ width: 56%;
+ height: 40px;
+ margin-left: 20px;
+ font-size: 14px;
+ font-weight: 300;
+ text-align: left;
}
.editButtonContainer {

diff:
@@ -24,10 +24,10 @@ describe('Dialog Selector Test', () => {
/>);
// Number of Logos should match number of images
- expect(selector.find('.logo').length).toEqual(Object.keys(DIALECTS).length);
+ expect(selector.find('.dialectSelector .Select-option').length).toEqual(Object.keys(DIALECTS).length);
// Dialect not selected so should not be found
- expect(selector.find('.logoSelected').length).toBe(0);
+ expect(selector.find('.dialectSelector .Select-option .is-selected').length).toBe(0);
});
it('should verify Dialect Selector created with dialect', () => {
@@ -42,6 +42,7 @@ describe('Dialog Selector Test', () => {
/>);
// Dialect not selected so should not be found
- expect(selector.find('.logoSelected').first().key()).toBe(connectionObject.dialect);
+ expect(selector.find('.dialectSelector .Select-option .is-selected')
+ .first().key()).toBe(connectionObject.dialect);
});
});

original_message: add drop down connection select


message: style API usage

diff:
@@ -470,7 +470,8 @@ export class PreviewModal extends Component {
<Timestamp value={query.nextScheduledAt} />{' '}
{canEdit && (
<span style={{paddingLeft: '7px'}}>
- (<Link
+ (
+ <Link
className="refresh-button"
style={{color: '#506784'}}
disabled={this.state.loading}
@@ -481,7 +482,8 @@ export class PreviewModal extends Component {
: this.state.confirmedRun
? 'are you sure?'
: 'run now'}
- </Link>)
+ </Link>
+ )
</span>
)}
</div>

diff:
@@ -20,8 +20,8 @@ export const AdditionalCallsPreview = props => {
const newTotal = additionalCalls + currTotal;
return (
-

- API usage: {additionalCalls} calls/day (new total: {formatUsageRatio(newTotal)})
+

+ API Usage: {additionalCalls} calls/day (new total: {formatUsageRatio(newTotal)})

);
};
@@ -35,11 +35,7 @@ AdditionalCallsPreview.defaultProps = {
};
export const IndividualCallCount = ({count}) => {
- return (
-
- {pluralize('API call', count, true)} per day
-
- );
+ return {pluralize('API call', count, true)} per day;
};
IndividualCallCount.propTypes = {
count: PropTypes.number.isRequired
@@ -48,8 +44,10 @@ IndividualCallCount.propTypes = {
export const CallCountWidget = ({count}) => {
return (
<React.Fragment>
-
API Usage

-
{formatUsageRatio(count)} API calls

+

+
API Usage

+ {formatUsageRatio(count)} API calls
+

</React.Fragment>
);
};

diff:
@@ -536,9 +536,9 @@ class Scheduler extends Component {
<React.Fragment>
<Row
style={{
- marginTop: 24,
- marginBottom: 24,
- justifyContent: 'space-between'
+ marginBottom: 8,
+ justifyContent: 'space-between',
+ alignItems: 'center'
}}
>
<input
@@ -551,7 +551,7 @@ class Scheduler extends Component {
Create Scheduled Query
</button>
)}
-

+

<CallCountWidget count={totalCallsPerDay} />

</Row>

diff:
@@ -23,9 +23,8 @@ const CreateModal = require('../../../../../app/components/Settings/scheduler/mo
const ErrorComponent = require('../../../../../app/components/error.jsx');
const CronPicker = require('../../../../../app/components/Settings/cron-picker/cron-picker.jsx');
const TagPicker = require('../../../../../app/components/Settings/scheduler/pickers/tag-picker').default;
-const AdditionalCallsPreview = require(
- '../../../../../app/components/Settings/scheduler/presentational/api-call-counts.jsx'
-).AdditionalCallsPreview;
+const AdditionalCallsPreview = require('../../../../../app/components/Settings/scheduler/presentational/api-call-counts.jsx')
+ .AdditionalCallsPreview;
describe('Create Modal Test', () => {
beforeAll(() => {
@@ -168,7 +167,7 @@ describe('Create Modal Test', () => {
);
expect(component.find(AdditionalCallsPreview).text()).toBe(
- 'API usage: 288 calls/day (new total: 1288 / 10000)'
+ 'API Usage: 288 calls/day (new total: 1288 / 10000)'
);
component = mount(
@@ -185,8 +184,7 @@ describe('Create Modal Test', () => {
.find(AdditionalCallsPreview)
.find('span')
.last()
- .prop('style')
- .color
+ .prop('style').color
).toBe('#EF553B');
});
});

original_message: style API usage


message: move clear search, adjust api usage visual hierarchy

diff:
@@ -10,8 +10,8 @@ export const AdditionalCallsPreview = props => {
return (

- API Usage: {formatCalls(additionalCalls)} {pluralize('call', additionalCalls)}/day (new total:{' '}
- {formatCalls(newTotal)})
+ API Usage: {formatCalls(additionalCalls)} {pluralize('call', additionalCalls)}
+ /day (new total: {formatCalls(newTotal)})

);
};
@@ -36,10 +36,20 @@ IndividualCallCount.propTypes = {
export const CallCountWidget = ({count}) => {
return (
<React.Fragment>
-

- API Usage
- {formatCalls(count)} {pluralize('call', count)}/day
- for this connection
+

+
+ API USAGE
+
+
+ {formatCalls(count)} {pluralize('call', count)}
+ /day
+
+ (for this connection)

</React.Fragment>
);

diff:
@@ -556,21 +556,36 @@ class Scheduler extends Component {
<Row
style={{
marginBottom: 8,
- justifyContent: 'space-between',
alignItems: 'center'
}}
>
<input
+ style={{width: '40%'}}
value={this.state.search}
onChange={this.handleSearchChange}
placeholder="Search scheduled queries..."
/>
+ <Column style={{width: 'unset', marginLeft: 16}}>
+ {this.state.search && (
+
+ Clear
+
+ )}
+ </Column>
{!contains(this.props.dialect, SQL_DIALECTS_USING_EDITOR) && (
<button style={{marginRight: '16px'}} onClick={this.openCreateModal}>
Create Scheduled Query
</button>
)}
-

+

<CallCountWidget count={totalCallsPerDay} />

</Row>
@@ -649,23 +664,8 @@ class Scheduler extends Component {

</Column>
- <Column style={{width: '100%', maxWidth: 720}}>
+ <Column style={{width: '100%', maxWidth: 560}}>
<Row style={{width: 'auto', justifyContent: 'flex-end'}}>
- <Column style={{maxWidth: 168, marginRight: 16}}>
- {this.state.search && (
-
- Clear current search
-
- )}
- </Column>
<Column
style={{
position: 'relative',


message: Examples for merkle storage article

diff:
@@ -48,6 +48,7 @@ use std::convert::TryInto;
use std::hash::Hash;
use std::sync::Arc;
use std::time::Instant;
+use hex;
use blake2::digest::{Update, VariableOutput};
use blake2::VarBlake2b;
@@ -269,6 +270,13 @@ fn encode_irmin_node_kind(kind: &NodeKind) -> [u8; 8] {
}
}
+// Calculates hash of tree
+// uses BLAKE2 binary 256 length hash function
+// hash is calculated as:
+// <number of child nodes (8 bytes)><CHILD NODE>
+// where:
+// - CHILD NODE - <NODE TYPE><length of string (1 byte)><string/path bytes><length of hash (8bytes)><hash bytes>
+// - NODE TYPE - leaf node(0xff0000000000000000) or internal node (0x0000000000000000)
fn hash_tree(tree: &Tree) -> Result<EntryHash, MerkleError> {
let mut hasher = VarBlake2b::new(HASH_LEN).unwrap();
@@ -284,6 +292,46 @@ fn hash_tree(tree: &Tree) -> Result<EntryHash, MerkleError> {
Ok(hasher.finalize_boxed().as_ref().try_into()?)
}
+// Calculates hash of BLOB
+// uses BLAKE2 binary 256 length hash function
+// hash is calculated as <length of data (8 bytes)><data>
+fn hash_blob(blob: &ContextValue) -> Result<EntryHash, MerkleError> {
+ let mut hasher = VarBlake2b::new(HASH_LEN).unwrap();
+ hasher.update(&(blob.len() as u64).to_be_bytes());
+ hasher.update(blob);
+
+ Ok(hasher.finalize_boxed().as_ref().try_into()?)
+}
+
+// Calculates hash of commit
+// uses BLAKE2 binary 256 length hash function
+// hash is calculated as:
+// <hash length (8 bytes)><tree hash bytes>
+// <length of parent hash (8bytes)><parent hash bytes>
+// <time in epoch format (8bytes)
+// <commit author name length (8bytes)><commit author name bytes>
+// <commit message length (8bytes)><commit message bytes>
+fn hash_commit(commit: &Commit) -> Result<EntryHash, MerkleError> {
+ let mut hasher = VarBlake2b::new(HASH_LEN).unwrap();
+ hasher.update(&(HASH_LEN as u64).to_be_bytes());
+ hasher.update(&commit.root_hash);
+
+ if commit.parent_commit_hash.is_none() {
+ hasher.update(&(0_u64).to_be_bytes());
+ } else {
+ hasher.update(&(1_u64).to_be_bytes()); // # of parents; we support only 1
+ hasher.update(&(commit.parent_commit_hash.unwrap().len() as u64).to_be_bytes());
+ hasher.update(&commit.parent_commit_hash.unwrap());
+ }
+ hasher.update(&(commit.time as u64).to_be_bytes());
+ hasher.update(&(commit.author.len() as u64).to_be_bytes());
+ hasher.update(&commit.author.clone().into_bytes());
+ hasher.update(&(commit.message.len() as u64).to_be_bytes());
+ hasher.update(&commit.message.clone().into_bytes());
+
+ Ok(hasher.finalize_boxed().as_ref().try_into()?)
+}
+
impl MerkleStorage {
pub fn new(db: Arc<MerkleStorageKV>) -> Self {
MerkleStorage {
@@ -583,11 +631,11 @@ impl MerkleStorage {
};
let entry = Entry::Commit(new_commit.clone());
- self.put_to_staging_area(&self.hash_commit(&new_commit)?, entry.clone())?;
+ self.put_to_staging_area(&hash_commit(&new_commit)?, entry.clone())?;
self.persist_staged_entry_to_db(&entry)?;
self.staged = Vec::new();
self.staged_indices = HashMap::new();
- let last_commit_hash = self.hash_commit(&new_commit)?;
+ let last_commit_hash = hash_commit(&new_commit)?;
self.last_commit_hash = Some(last_commit_hash);
self.update_execution_stats("Commit".to_string(), None, &instant);
@@ -677,7 +725,7 @@ impl MerkleStorage {
Action::Set(set) => {
let root_hash = self.current_stage_tree_hash.unwrap();
let key = &set.key;
- let blob_hash = self.hash_blob(&set.value)?;
+ let blob_hash = hash_blob(&set.value)?;
self.put_to_staging_area(&blob_hash, Entry::Blob(set.value.clone()))?;
let new_node = Node { entry_hash: blob_hash, node_kind: NodeKind::Leaf };
@@ -1058,41 +1106,12 @@ impl MerkleStorage {
fn hash_entry(&self, entry: &Entry) -> Result<EntryHash, MerkleError> {
match entry {
- Entry::Commit(commit) => self.hash_commit(&commit),
+ Entry::Commit(commit) => hash_commit(&commit),
Entry::Tree(tree) => hash_tree(&tree),
- Entry::Blob(blob) => self.hash_blob(blob),
+ Entry::Blob(blob) => hash_blob(blob),
}
}
- fn hash_commit(&self, commit: &Commit) -> Result<EntryHash, MerkleError> {
- let mut hasher = VarBlake2b::new(HASH_LEN).unwrap();
- hasher.update(&(HASH_LEN as u64).to_be_bytes());
- hasher.update(&commit.root_hash);
-
- if commit.parent_commit_hash.is_none() {
- hasher.update(&(0_u64).to_be_bytes());
- } else {
- hasher.update(&(1_u64).to_be_bytes()); // # of parents; we support only 1
- hasher.update(&(commit.parent_commit_hash.unwrap().len() as u64).to_be_bytes());
- hasher.update(&commit.parent_commit_hash.unwrap());
- }
- hasher.update(&(commit.time as u64).to_be_bytes());
- hasher.update(&(commit.author.len() as u64).to_be_bytes());
- hasher.update(&commit.author.clone().into_bytes());
- hasher.update(&(commit.message.len() as u64).to_be_bytes());
- hasher.update(&commit.message.clone().into_bytes());
-
- Ok(hasher.finalize_boxed().as_ref().try_into()?)
- }
-
- fn hash_blob(&self, blob: &ContextValue) -> Result<EntryHash, MerkleError> {
- let mut hasher = VarBlake2b::new(HASH_LEN).unwrap();
- hasher.update(&(blob.len() as u64).to_be_bytes());
- hasher.update(blob);
-
- Ok(hasher.finalize_boxed().as_ref().try_into()?)
- }
-
fn get_tree(&self, hash: &EntryHash) -> Result<Tree, MerkleError> {
match self.get_entry(hash)? {
Entry::Tree(tree) => Ok(tree),
@@ -1163,6 +1182,40 @@ impl MerkleStorage {
self.last_commit_hash
}
+ fn get_staged_entries(&self) -> std::string::String{
+ let mut result =String::new();
+ for (hash, _,entry) in &self.staged{
+ match entry{
+ Entry::Blob(blob) => {
+ result += &format!("{}: Value {:?}, \n", hex::encode(&hash[0..3]), blob);
+ }
+
+ Entry::Tree(tree) => {
+ if tree.is_empty(){
+ continue;
+ }
+ let tree_hash = &hash_tree(tree).unwrap()[0..3];
+ result += &format!("{}: Tree {{", hex::encode(tree_hash));
+
+ for (path, val) in tree {
+ let kind = if let NodeKind::NonLeaf = val.node_kind {
+ "Tree"
+ }else{
+ "Value/Leaf"
+ };
+ result += &format!("{}: {}({:?}), ", path, kind, hex::encode(&val.entry_hash[0..3]));
+ }
+ result += "}}\n";
+ }
+
+ Entry::Commit(_) => {
+ panic!("commits must not occur in staged area");
+ }
+ }
+ }
+ return result;
+ }
+
/// Get various merkle storage statistics
pub fn get_merkle_stats(&self) -> Result<MerkleStorageStats, MerkleError> {
let db_stats = self.db.get_mem_use_stats()?;
@@ -1289,6 +1342,175 @@ mod tests {
assert_eq!(storage.get_history(&commit, c_foo).unwrap(), vec![3, 4]);
}
+ #[test]
+ fn test_hash_of_value_1_blob() {
+ let expected_hash = "407f958990678e2e9fb06758bc6520dae46d838d39948a4c51a5b19bd079293d".to_string();
+
+ // hash is calculated as <length of data (8 bytes)><data>
+ let blob = vec![1];
+
+ // hexademical representation of above commit:
+ //
+ // value length (8 bytes) -> 00 00 00 00 00 00 00 01
+ // value bytes -> 01
+
+ let mut bytes = String::new();
+ let value_length = "0000000000000001";
+ let value = "01";
+
+ println!("calculating hash of value: {:?}\n", blob);
+
+ println!("[hex] value_length : {}", value_length);
+ println!("[hex] value : {}", value);
+
+ bytes += &value_length;
+ bytes += &value;
+
+ println!("manually calculated haxedemical representation of value/blob: {}", bytes);
+
+ let mut hasher= VarBlake2b::new(HASH_LEN).unwrap();
+ hasher.update(hex::decode(bytes).unwrap());
+ let calcualted_hash = hex::encode(hasher.finalize_boxed().as_ref());
+
+ println!("calculated hash of the value/blob: {}", calcualted_hash);
+
+ assert_eq!(expected_hash, calcualted_hash);
+ assert_eq!(hex::encode(hash_blob(&blob).unwrap()), calcualted_hash);
+ }
+
+ #[test]
+ fn test_hash_of_commit() {
+ // Calculates hash of commit
+ // uses BLAKE2 binary 256 length hash function
+ // hash is calculated as:
+ // <hash length (8 bytes)><tree hash bytes>
+ // <length of parent hash (8bytes)><parent hash bytes>
+ // <time in epoch format (8bytes)
+ // <commit author name length (8bytes)><commit author name bytes>
+ // <commit message length (8bytes)><commit message bytes>
+ let expected_commit_hash = "e6de3fd37b1dc2b3c9d072ea67c2c5be1b55eeed9f5377b2bfc1228e6f9cb69b";
+ let dummy_commit = Commit{
+ parent_commit_hash: None,
+ root_hash: hex::decode("0d78b30e959c2a079e8ccb4ca19d428c95d29b2f02a35c1c58ef9c8972bc26aa").unwrap().try_into().unwrap(),
+ time: 0,
+ author: "Tezedge".to_string(),
+ message: "persist changes".to_string(),
+ };
+
+ // hexademical representation of above commit:
+ //
+ // hash length (8 bytes) -> 00 00 00 00 00 00 00 20
+ // tree hash bytes -> 0d78b30e959c2a079e8ccb4ca19d428c95d29b2f02a35c1c58ef9c8972bc26aa
+ // parents count -> 00 00 00 00 00 00 00 00 (0)
+ // commit time -> 00 00 00 00 00 00 00 00 (0)
+ // commit author name length -> 00 00 00 00 00 00 00 07 (7)
+ // commit author name ('Tezedge') -> 54 65 7a 65 64 67 65 (Tezedge)
+ // commit message length -> 00 00 00 00 00 00 00 0xf (15)
+
+ let mut bytes = String::new();
+ let hash_length = "0000000000000020"; // 32
+ let tree_hash = "0d78b30e959c2a079e8ccb4ca19d428c95d29b2f02a35c1c58ef9c8972bc26aa"; // tree hash bytes
+ let parents_count = "0000000000000000"; // 0
+ let commit_time = "0000000000000000"; // 0
+ let commit_author_name_length = "0000000000000007"; // 7
+ let commit_author_name = "54657a65646765"; // 'Tezedge'
+ let commit_message_length = "000000000000000f"; // 15
+ let commit_message = "70657273697374206368616e676573"; // 'persist changes'
+
+ println!("calculating hash of commit: \n\t{:?}\n", dummy_commit);
+
+ println!("[hex] hash_length : {}", hash_length);
+ println!("[hex] tree_hash : {}", tree_hash);
+ println!("[hex] parents_count : {}", parents_count);
+ println!("[hex] commit_time : {}", commit_time);
+ println!("[hex] commit_author_name_length : {}", commit_author_name_length);
+ println!("[hex] commit_author_name : {}", commit_author_name);
+ println!("[hex] commit_message_length : {}", commit_message_length);
+ println!("[hex] commit_message : {}", commit_message);
+
+ bytes += &hash_length;
+ bytes += &tree_hash;
+ bytes += &parents_count;
+ bytes += &commit_time;
+ bytes += &commit_author_name_length;
+ bytes += &commit_author_name;
+ bytes += &commit_message_length;
+ bytes += &commit_message;
+
+ println!("manually calculated haxedemical representation of commit: {}", bytes);
+
+ let mut hasher = VarBlake2b::new(HASH_LEN).unwrap();
+ hasher.update(hex::decode(bytes).unwrap());
+ let calculated_commit_hash = hasher.finalize_boxed();
+
+ println!("calculated hash of the commit: {}", hex::encode(calculated_commit_hash.as_ref()));
+
+ assert_eq!(calculated_commit_hash.as_ref(), hash_commit(&dummy_commit).unwrap());
+ assert_eq!(expected_commit_hash, hex::encode(calculated_commit_hash.as_ref()));
+ }
+
+ #[test]
+ fn test_hash_of_small_tree() {
+ // Calculates hash of tree
+ // uses BLAKE2 binary 256 length hash function
+ // hash is calculated as:
+ // <number of child nodes (8 bytes)><CHILD NODE>
+ // where:
+ // - CHILD NODE - <NODE TYPE><length of string (1 byte)><string/path bytes><length of hash (8bytes)><hash bytes>
+ // - NODE TYPE - leaf node(0xff00000000000000) or internal node (0x0000000000000000)
+
+ let expected_tree_hash = "d49a53323107f2ae40b01eaa4e9bec4d02801daf60bab82dc2529e40d40fa917";
+ let mut dummy_tree = Tree::new();
+ let node = Node{
+ node_kind: NodeKind::Leaf,
+ entry_hash: hash_blob(&vec![1]).unwrap() // 407f958990678e2e9fb06758bc6520dae46d838d39948a4c51a5b19bd079293d
+ };
+ dummy_tree.insert("a".to_string(), node);
+
+ // hexademical representation of above tree:
+ //
+ // number of child nodes -> 00 00 00 00 00 00 00 01 (1)
+ // node type -> ff 00 00 00 00 00 00 00 (leaf node)
+ // length of string -> 01 (1)
+ // string -> 61 ('a')
+ // length of hash -> 00 00 00 00 00 00 00 20 (32)
+ // hash -> 407f958990678e2e9fb06758bc6520dae46d838d39948a4c51a5b19bd079293d
+
+ let mut bytes = String::new();
+ let child_nodes = "0000000000000001";
+ let leaf_node = "ff00000000000000";
+ let string_length = "01";
+ let string_value = "61";
+ let hash_length = "0000000000000020";
+ let hash = "407f958990678e2e9fb06758bc6520dae46d838d39948a4c51a5b19bd079293d";
+
+ println!("calculating hash of tree: \n\t{:?}\n", dummy_tree);
+ println!("[hex] child nodes count: {}", child_nodes);
+ println!("[hex] leaf_node : {}", leaf_node);
+ println!("[hex] string_length : {}", string_length);
+ println!("[hex] string_value : {}", string_value);
+ println!("[hex] hash_length : {}", hash_length);
+ println!("[hex] hash : {}", hash);
+
+ bytes += &child_nodes;
+ bytes += &leaf_node;
+ bytes += &string_length;
+ bytes += &string_value;
+ bytes += &hash_length;
+ bytes += &hash;
+
+ println!("manually calculated haxedemical representation of tree: {}", bytes);
+
+ let mut hasher = VarBlake2b::new(HASH_LEN).unwrap();
+ hasher.update(hex::decode(bytes).unwrap());
+ let calculated_tree_hash = hasher.finalize_boxed();
+
+ println!("calculated hash of the tree: {}", hex::encode(calculated_tree_hash.as_ref()));
+
+ assert_eq!(calculated_tree_hash.as_ref(), hash_tree(&dummy_tree).unwrap());
+ assert_eq!(calculated_tree_hash.as_ref(), hex::decode(expected_tree_hash).unwrap());
+ }
+
#[test]
fn test_tree_hash() {
let cache = Cache::new_lru_cache(32 * 1024 * 1024).unwrap();
@@ -1326,6 +1548,64 @@ mod tests {
// full irmin hash: ca7bc7022ffbd35acc97f7defb00c486bb7f4d19a2d62790d5949775eb74f3c8
}
+ fn get_short_hash(hash: &EntryHash) -> String{
+ return hex::encode(&hash[0..3]);
+ }
+
+ fn get_staged_root_short_hash(storage: &mut MerkleStorage) -> String{
+ let tree = storage.get_staged_root().unwrap();
+ let hash = hash_tree(&tree).unwrap();
+ return get_short_hash(&hash);
+ }
+
+ #[test]
+ fn test_examples_from_article_about_storage() {
+ let cache = Cache::new_lru_cache(32 * 1024 * 1024).unwrap();
+ let db_name = "test_examples_from_article_about_storage";
+ clean_db(db_name);
+ let mut storage = get_storage(db_name, &cache);
+
+ storage.set(&vec!["a".to_string()], &vec![1]);
+ storage.apply_actions_to_staging_area();
+ let root = get_staged_root_short_hash(& mut storage);
+ println!("SET [a] = 1\nROOT: {}", root);
+ println!("CONTENT {}", storage.get_staged_entries());
+ assert_eq!(root, "d49a53".to_string());
+
+ storage.set(&vec!["b".to_string(), "c".to_string()], &vec![1]);
+ storage.apply_actions_to_staging_area();
+ let root = get_staged_root_short_hash(& mut storage);
+ println!("\nSET [b,c] = 1\nROOT: {}", root);
+ print!("{}", storage.get_staged_entries());
+ assert_eq!(root, "ed8adf".to_string());
+
+ storage.set(&vec!["b".to_string(), "d".to_string()], &vec![2]);
+ storage.apply_actions_to_staging_area();
+ let root = get_staged_root_short_hash(& mut storage);
+ println!("\nSET [b,d] = 2\nROOT: {}", root);
+ print!("{}", storage.get_staged_entries());
+ assert_eq!(root, "437186".to_string());
+
+ storage.set(&vec!["a".to_string()], &vec![2]);
+ storage.apply_actions_to_staging_area();
+ let root = get_staged_root_short_hash(& mut storage);
+ println!("\nSET [a] = 2\nROOT: {}", root);
+ print!("{}", storage.get_staged_entries());
+ assert_eq!(root, "0d78b3".to_string());
+
+
+ let entries = storage.get_staged_entries();
+ let commit_hash = storage.commit(0, "Tezedge".to_string(), "persist changes".to_string()).unwrap();
+ storage.apply_actions_to_staging_area();
+ println!("\nCOMMIT time:0 author:'tezedge' message:'persist'");
+ println!("ROOT: {}", get_short_hash(&commit_hash));
+ if let Entry::Commit(c) = storage.get_entry(&commit_hash).unwrap() {
+ println!("{} : Commit{{time:{}, message:{}, author:{}, root_hash:{}, parent_commit_hash: None}}", get_short_hash(&commit_hash), c.time, c.message, c.author, get_short_hash(&c.root_hash));
+ }
+ print!("{}", entries);
+ assert_eq!("e6de3f", get_short_hash(&commit_hash))
+ }
+
#[test]
fn test_multiple_commit_hash() {
let cache = Cache::new_lru_cache(32 * 1024 * 1024).unwrap();

repo: tezedge/tezedge

original_message: Examples for merkle storage article


message: clippy linter fixes

diff:
@@ -626,11 +626,11 @@ mod tests {
.unwrap();
let index = BlockByLevelIndex::new(Arc::new(db));
- for i in vec![1161, 66441, 905, 66185, 649, 65929, 393, 65673] {
+ for i in [1161, 66441, 905, 66185, 649, 65929, 393, 65673].iter() {
index.put(
- i,
+ *i,
&BlockStorageColumnsLocation {
- block_header: Location::new(i as u64),
+ block_header: Location::new(*i as u64),
block_json_data: None,
block_additional_data: None,
},

diff:
@@ -133,13 +133,11 @@ impl ContextActionStorage {
sorted_intersect::sorted_intersect(iterators, limit.unwrap_or(std::usize::MAX))
.into_iter(),
)
- } else {
- if let Some(limit) = limit {
+ } else if let Some(limit) = limit {
self.load_indexes(base_iterator.take(limit))
} else {
self.load_indexes(base_iterator)
}
- }
} else {
let mut base_iterator = self
.context_by_contract_index
@@ -162,15 +160,13 @@ impl ContextActionStorage {
} else {
Ok(Default::default())
}
- } else {
- if let Some(limit) = limit {
+ } else if let Some(limit) = limit {
self.load_indexes(base_iterator.take(limit))
} else {
self.load_indexes(base_iterator)
}
}
}
- }
#[inline]
pub fn get_by_block_hash(
@@ -694,11 +690,11 @@ impl ContextActionByTypeIndex {
}
#[inline]
- fn get_by_action_type_iterator<'a>(
- &'a self,
+ fn get_by_action_type_iterator(
+ &self,
action_type: ContextActionType,
cursor_id: Option<SequenceNumber>,
- ) -> Result<impl Iterator<Item = u64> + 'a, StorageError> {
+ ) -> Result<impl Iterator<Item = u64> + '_, StorageError> {
let iterate_from_key = cursor_id.map_or_else(
|| ContextActionByTypeIndexKey::from_action_type_prefix(action_type),
|cursor_id| ContextActionByTypeIndexKey::new(action_type, cursor_id),
@@ -996,11 +992,7 @@ pub mod sorted_intersect {
})
});
- if let Some((_, true)) = value {
- true
- } else {
- false
- }
+ matches!(value, Some((_, true)))
}
}
@@ -1020,7 +1012,8 @@ mod tests {
};
let encoded_bytes = expected.encode()?;
let decoded = ContextActionByContractIndexKey::decode(&encoded_bytes)?;
- Ok(assert_eq!(expected, decoded))
+ assert_eq!(expected, decoded);
+ Ok(())
}
#[test]
@@ -1031,7 +1024,8 @@ mod tests {
};
let encoded_bytes = expected.encode()?;
let decoded = ContextActionByBlockHashKey::decode(&encoded_bytes)?;
- Ok(assert_eq!(expected, decoded))
+ assert_eq!(expected, decoded);
+ Ok(())
}
#[test]
@@ -1042,7 +1036,8 @@ mod tests {
};
let encoded_bytes = expected.encode()?;
let decoded = ContextActionByBlockHashKey::decode(&encoded_bytes)?;
- Ok(assert_eq!(expected, decoded))
+ assert_eq!(expected, decoded);
+ Ok(())
}
#[test]
@@ -1058,10 +1053,11 @@ mod tests {
}
.encode()?;
- Ok(assert_eq!(
+ assert_eq!(
Ordering::Less,
ContextActionByContractIndexKey::reverse_id_comparator(&a, &b)
- ))
+ );
+ Ok(())
}
#[test]

diff:
//! ``
//!
//! Reference: https://git-scm.com/book/en/v2/Git-Internals-Git-Objects
-use hex;
use std::array::TryFromSliceError;
+use std::collections::hash_map::Entry as MapEntry;
use std::collections::{BTreeMap, HashMap};
use std::convert::TryInto;
use std::hash::Hash;
@@ -955,16 +955,18 @@ impl MerkleStorage {
let tree_is_empty = tree.is_empty();
- // Add mapping from hash to index
- if self.staged_indices.contains_key(&new_tree_hash) {
+ match self.staged_indices.entry(new_tree_hash) {
+ MapEntry::Occupied(_) => {
// entry already exists in staging, increase its refcnt only
// staged_indices will point to the other entry while this entry will be
// unreachable (wasting space, but removing it would require changing all
// other indices)
self.increase_refcnt_for_staging_entry(&new_tree_hash)?;
- } else {
+ }
+ MapEntry::Vacant(_) => {
self.staged_indices.insert(new_tree_hash, idx);
}
+ }
if tree_is_empty {
// last element was removed, delete this node
@@ -1267,7 +1269,7 @@ impl MerkleStorage {
self.last_commit_hash
}
- fn get_staged_entries(&self) -> std::string::String {
+ pub fn get_staged_entries(&self) -> std::string::String {
let mut result = String::new();
for (hash, _, entry) in &self.staged {
match entry {
@@ -1303,7 +1305,7 @@ impl MerkleStorage {
}
}
}
- return result;
+ result
}
/// Get various merkle storage statistics
@@ -1396,11 +1398,11 @@ impl MerkleStorage {
#[cfg(test)]
#[allow(unused_must_use)]
mod tests {
- use std::path::{Path, PathBuf};
- use std::{env, fs};
-
+ // use hex;
use assert_json_diff::assert_json_eq;
use rocksdb::{Options, DB};
+ use std::path::{Path, PathBuf};
+ use std::{env, fs};
use super::*;
@@ -1703,13 +1705,13 @@ mod tests {
}
fn get_short_hash(hash: &EntryHash) -> String {
- return hex::encode(&hash[0..3]);
+ hex::encode(&hash[0..3])
}
fn get_staged_root_short_hash(storage: &mut MerkleStorage) -> String {
let tree = storage.get_staged_root().unwrap();
let hash = hash_tree(&tree).unwrap();
- return get_short_hash(&hash);
+ get_short_hash(&hash)
}
#[test]

diff:
@@ -303,7 +303,8 @@ mod tests {
};
let encoded_bytes = expected.encode()?;
let decoded = Meta::decode(&encoded_bytes)?;
- Ok(assert_eq!(expected, decoded))
+ assert_eq!(expected, decoded);
+ Ok(())
}
#[test]

diff:
@@ -185,6 +185,7 @@ mod tests {
};
let encoded_bytes = expected.encode()?;
let decoded = OperationKey::decode(&encoded_bytes)?;
- Ok(assert_eq!(expected, decoded))
+ assert_eq!(expected, decoded);
+ Ok(())
}
}

diff:
// Copyright (c) SimpleStaking and Tezedge Contributors
// SPDX-License-Identifier: MIT
+use failure::Fail;
+use std::cmp::Ordering as Ord;
use std::collections::HashMap;
use std::sync::atomic::{AtomicI32, AtomicU64, Ordering};
use std::sync::{Arc, Condvar, Mutex, PoisonError};
-use failure::Fail;
-
use crate::persistent::{DBError, KeyValueSchema, KeyValueStoreWithSchema};
/// Provider a system wide unique sequence generators backed by a permanent storage.
@@ -97,11 +97,13 @@ impl SequenceGenerator {
let seq = loop {
let available = self.seq_available.fetch_add(-1, Ordering::SeqCst);
- if available > 0 {
+ match available.cmp(&0) {
+ Ord::Greater => {
// no need to allocate new sequence numbers yet
let seq = self.seq_cur.fetch_add(1, Ordering::SeqCst);
break seq;
- } else if available == 0 {
+ }
+ Ord::Equal => {
// last pre-allocated sequence numbers was allocated now, we have to perform a new allocation
let seq = self.seq_cur.fetch_add(1, Ordering::SeqCst);
@@ -121,12 +123,14 @@ impl SequenceGenerator {
self.guard.1.notify_all();
break seq;
- } else {
+ }
+ Ord::Less => {
// wait until seq_available is positive number again
let _lock = self.guard.1.wait_while(self.guard.0.lock()?, |_| {
self.seq_available.load(Ordering::SeqCst) <= 0
})?;
}
+ }
};
Ok(seq)

change_type: RENAME

diff:
@@ -49,7 +49,7 @@ impl DatabaseBackedSkipList {
let list_db: Arc<SkipListDatabase> = db;
let state = list_db
.get(&list_id)?
- .unwrap_or_else(|| SkipListState { levels: 1, len: 0 });
+ .unwrap_or(SkipListState { levels: 1, len: 0 });
Ok(Self {
list_db,
@@ -154,6 +154,8 @@ pub trait SkipList {
fn levels(&self) -> usize;
+ fn is_empty(&self) -> bool;
+
fn contains(&self, index: usize) -> bool;
}
@@ -164,6 +166,11 @@ impl SkipList for DatabaseBackedSkipList {
self.state.len
}
+ #[inline]
+ fn is_empty(&self) -> bool {
+ self.state.len > 0
+ }
+
#[inline]
fn levels(&self) -> usize {
self.state.levels
@@ -218,20 +225,16 @@ where
loop {
if let Some(value) = lane.get(pos.index(), key)? as Option<V> {
return Ok(Some(value));
- } else {
- if pos.is_edge_node() && pos.level() < highest_level {
+ } else if pos.is_edge_node() && pos.level() < highest_level {
pos = pos.higher();
lane = lane.higher_lane();
- } else {
- if pos.index() == 0 {
+ } else if pos.index() == 0 {
return Ok(None);
} else {
pos = pos.prev();
}
}
}
- }
- }
/// Push new value into the end of the list. Beware, this is operation is
/// not thread safe and should be handled with care !!!

diff:
pub use crate::skip_list::content::{Bucket, ListValue, SkipListError};
pub use crate::skip_list::lane::{Lane, TypedLane};
-pub use crate::skip_list::skip_list::{DatabaseBackedSkipList, SkipList, TypedSkipList};
+pub use crate::skip_list::list::{DatabaseBackedSkipList, SkipList, TypedSkipList};
mod content;
mod lane;
-mod skip_list;
+mod list;
pub(crate) const LEVEL_BASE: usize = 8;

original_message: clippy linter fixes


diff:
@@ -189,7 +189,6 @@ impl FileAppender {
if self.rotate_compress {
let (plain_path, temp_gz_path) = self.rotated_paths_for_compression()?;
let (tx, rx) = mpsc::channel();
-
fs::rename(&self.path, &plain_path)?;
thread::spawn(move || {
let result = Self::compress(plain_path, temp_gz_path, rotated_path);
@@ -276,6 +275,19 @@ impl Write for FileAppender {
if self.written_size >= self.rotate_size {
self.rotate()?;
}
+
+ if let Some(receiver) = &self.wait_compression {
+ let result = receiver.recv().map_err(|_| {
+ io::Error::new(io::ErrorKind::Other, "Log file compression thread aborted")
+ })?;
+ if let Err(_) = result {
+ return Err(io::Error::new(
+ io::ErrorKind::Other,
+ "Log file compression failed",
+ ));
+ }
+ self.wait_compression = None;
+ }
Ok(())
}
}
@@ -292,9 +304,6 @@ fn default_rotate_keep() -> usize {
#[cfg(test)]
mod tests {
- use std::thread;
- use std::time::Duration;
-
use tempfile::{Builder as TempDirBuilder, TempDir};
use super::*;
@@ -315,26 +324,26 @@ mod tests {
.build();
write!(appender, "hello");
- thread::sleep(Duration::from_millis(50));
+ appender.flush().unwrap();
assert!(dir.path().join("foo.log").exists());
assert!(!dir.path().join("foo.log.1").exists());
write!(appender, "world");
- thread::sleep(Duration::from_millis(50));
+ appender.flush().unwrap();
assert!(dir.path().join("foo.log").exists());
assert!(!dir.path().join("foo.log.1").exists());
assert!(!dir.path().join("foo.log.2").exists());
assert!(!dir.path().join("foo.log.3").exists());
write!(appender, format!("vec(0): {:?}", vec![0; 128]));
- thread::sleep(Duration::from_millis(50));
+ appender.flush().unwrap();
assert!(dir.path().join("foo.log").exists());
assert!(dir.path().join("foo.log.1").exists());
assert!(!dir.path().join("foo.log.2").exists());
assert!(!dir.path().join("foo.log.3").exists());
write!(appender, format!("vec(1): {:?}", vec![0; 128]));
- thread::sleep(Duration::from_millis(50));
+ appender.flush().unwrap();
assert!(dir.path().join("foo.log").exists());
assert!(dir.path().join("foo.log.1").exists());
assert!(dir.path().join("foo.log.2").exists());
@@ -351,25 +360,25 @@ mod tests {
.build();
write!(appender, "hello");
- thread::sleep(Duration::from_millis(50));
+ appender.flush().unwrap();
assert!(dir.path().join("foo.log").exists());
assert!(!dir.path().join("foo.log.1").exists());
write!(appender, "world");
- thread::sleep(Duration::from_millis(50));
+ appender.flush().unwrap();
assert!(dir.path().join("foo.log").exists());
assert!(!dir.path().join("foo.log.1.gz").exists());
assert!(!dir.path().join("foo.log.2.gz").exists());
write!(appender, format!("vec(0): {:?}", vec![0; 128]));
- thread::sleep(Duration::from_millis(50));
+ appender.flush().unwrap();
assert!(dir.path().join("foo.log").exists());
assert!(dir.path().join("foo.log.1.gz").exists());
assert!(!dir.path().join("foo.log.2.gz").exists());
assert!(!dir.path().join("foo.log.3.gz").exists());
write!(appender, format!("vec(1): {:?}", vec![0; 128]));
- thread::sleep(Duration::from_millis(50));
+ appender.flush().unwrap();
assert!(dir.path().join("foo.log").exists());
assert!(dir.path().join("foo.log.1.gz").exists());
assert!(dir.path().join("foo.log.2.gz").exists());

original_message: [TE-299] wait for compression thread to complete in FileAppender::flush function


message: wait for spawned child processes

diff:
@@ -15,3 +15,17 @@ pub fn fork<F: FnOnce()>(child_func: F) -> libc::pid_t {
}
}
}
+
+pub fn wait(pid: libc::pid_t) -> bool {
+ // libc::waitpid is unsafe function
+ unsafe {
+ let mut status: i32 = 0;
+ let options: i32 = 0;
+ return match libc::waitpid(pid, &mut status as *mut i32, options) {
+ -1 => {
+ panic!("error occured libc::waitpid problem")
+ }
+ _pid => true,
+ };
+ }
+}

diff:
@@ -60,6 +60,8 @@ fn ipc_fork_and_client_exchange() -> Result<(), failure::Error> {
let recv = rx.receive()?;
assert_eq!(recv, "dog");
+ assert!(common::wait(child_pid));
+
Ok(())
}
@@ -130,7 +132,10 @@ fn ipc_fork_and_try_read_with_timeout() -> Result<(), failure::Error> {
Err(IpcError::ReceiveMessageTimeouted { .. }) => {
Err(format_err!("Unexpected IpcError::ReceiveMessageTimeouted"))
}
- Err(_) => Ok(()),
+ Err(_) => {
+ assert!(common::wait(child_pid));
+ Ok(())
+ }
Ok(_) => Err(format_err!("Unexpected result")),
}
}

original_message: wait for spawned child processes


message:
disable unused rocksdb zlib feature that is not working with address sanitizer
cleanup run.sh script, add new node-saddr which runs node with address sanitizer enabled
include docker environment for address sanitizer
include address sanitizer tests in github workflow

diff:
@@ -13,6 +13,7 @@ jobs:
strategy:
matrix:
os: [macos-latest, ubuntu-latest]
+ sanitizer: [none, address]
steps:
- uses: actions/checkout@v2
- name: Setup Rust
@@ -20,6 +21,12 @@ jobs:
with:
toolchain: nightly-2020-12-31
default: true
+ - name: Settings for cargo in Linux
+ if: matrix.os == 'ubuntu-latest' && matrix.sanitizer == 'address'
+ run: |
+ echo '[build]' >> ~/.cargo/config
+ echo 'rustflags = ["-Z", "sanitizer=address"]' >> ~/.cargo/config
+ echo 'rustdocflags = ["-Z", "sanitizer=address"]' >> ~/.cargo/config
- name: Settings for cargo in OSX
if: runner.os == 'macOS'
run: |
@@ -40,7 +47,15 @@ jobs:
uses: actions-rs/cargo@v1
with:
command: check
+ - name: cargo test SADDR
+ if: matrix.sanitizer == 'address'
+ uses: actions-rs/cargo@v1
+ with:
+ command: test
- name: cargo test
+ if: matrix.sanitizer == 'none'
uses: actions-rs/cargo@v1
with:
command: test
+
+

diff:
@@ -2012,8 +2012,7 @@ checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397"
[[package]]
name = "ocaml-interop"
version = "0.4.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73f6eb1378c49deceaf7fec407633c1f9ca8317ec58992da8a91e2c2164cc628"
+source = "git+https://github.com/simplestaking/ocaml-interop?branch=v0.4.4_saddr_fix#511d74095c4ba95408707e4ed4db7d0be8a73ab4"
dependencies = [
"ocaml-sys",
]

change_type: ADD

new_path: docker/sanitizer/Dockerfile

diff:
+FROM simplestakingcom/tezos-opam-builder:debian10 as build-env
+
+# Checkout and compile tezedge source code
+USER root
+
+ARG tezedge_git="https://github.com/simplestaking/tezedge.git"
+ARG rust_toolchain="nightly-2020-12-31"
+ENV RUSTUP_HOME=/opt/rustup
+ENV CARGO_HOME=/opt/cargo
+RUN mkdir $RUSTUP_HOME
+RUN mkdir $CARGO_HOME
+RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain ${rust_toolchain} -y
+ENV PATH=$CARGO_HOME/bin:$PATH
+ENV RUST_BACKTRACE=1
+ENV SODIUM_USE_PKG_CONFIG=1
+ENV OCAML_BUILD_CHAIN=remote
+
+#USER root
+RUN apt-get update && apt-get install clang libclang-dev libssl-dev -y
+RUN apt-get update && apt-get install gosu -y
+RUN echo "interceptor_via_lib:protocol-runner" > /asan.supp
+
+RUN rustup component add rust-src
+RUN chmod -R 777 /opt/cargo
+COPY docker-entrypoint.sh /entry.sh
+CMD ["/entry.sh"]

diff:
+#!/bin/bash -e
+REPO_ROOT=/home/dev/repo
+USER="dev"
+GROUP="dev"
+
+USERID="$(stat -c '%u' "$REPO_ROOT")"
+GROUPID="$(stat -c '%g' "$REPO_ROOT")"
+
+
+groupadd -g $GROUPID $GROUP || :
+
+useradd -o --uid $USERID --gid $GROUPID $USER
+cd /home/$USER/repo
+exec /usr/sbin/gosu "$USER" /bin/bash

diff:
+#!/bin/bash
+
+REPO_ROOT=$(readlink -f $(dirname $(readlink -f $0))/../../)
+#echo $REPO_ROOT
+
+docker build $REPO_ROOT/docker/sanitizer -t tezedge-sanitizer && docker run -v $REPO_ROOT:/home/dev/repo -it tezedge-sanitizer

diff:
@@ -12,7 +12,7 @@ futures = "0.3"
riker = "0.4"
rlimit = "0.5"
serde_json = "1.0"
-rocksdb = "0.15"
+rocksdb = {version = "0.15", features = ["snappy", "lz4", "zstd", "zlib"], default-features = false }
slog = { version = "2.7", features = ["max_level_trace"] }
slog-async = "2.6"
slog-json = "2.3"

diff:
# Copyright (c) SimpleStaking and Tezedge Contributors
# SPDX-License-Identifier: MIT
+# stop on first error
+set -e
+
warn_if_not_using_recommended_rust() {
RUSTC_TOOLCHAIN_VERSION="2020-12-31"
@@ -28,26 +31,51 @@ help() {
echo " # $0 docker --help"
}
-build_all() {
- # cargo build profile commandline argument
+configure_env_variables() {
+ options=$(getopt --longoptions debug,release,addrsanitizer --options "" --name $0 -- "$@")
+ eval set -- "$options"
+ while true ; do
case $1 in
- debug)
- # nothing to do here
+ --debug)
+ export PROFILE="debug"
shift
;;
- release)
- CARGO_PROFILE_ARG="--release"
+ --release)
+ export PROFILE="release"
+ shift
+ ;;
+ --addrsanitizer)
+ export RUSTFLAGS="-Z sanitizer=address"
+ export CARGO_BUILD_TARGET="x86_64-unknown-linux-gnu"
+ shift
+ ;;
+ --)
shift
+ break
;;
*)
echo "Invalid function argument"
exit 1
;;
esac
+ done
+}
+
+print_configuration() {
+ echo -ne "\033[1;37mRunning Tezedge node in ${PROFILE^^} mode"
+ if [ -n "$CARGO_BUILD_TARGET" ]; then
+ echo -ne "($CARGO_BUILD_TARGET)"
+ fi
+
+ if [ -n "$RUSTFLAGS" ]; then
+ echo -ne " extra compilation flags : '$RUSTFLAGS'"
+ fi
+ echo -ne "\e[0m\n"
+}
- # this is required for the most linux distributions
+build_all() {
export SODIUM_USE_PKG_CONFIG=1
- cargo build $CARGO_PROFILE_ARG || exit 1
+ cargo build
}
run_node() {
@@ -67,25 +95,6 @@ run_node() {
# rest of the commandline arguments will end up here
args=()
-
- # set compilation profile and cargo commandline argument
- PROFILE=""
- case $1 in
- debug)
- PROFILE="debug"
- shift
- ;;
- release)
- CARGO_PROFILE_ARG="--release"
- PROFILE="release"
- shift
- ;;
- *)
- echo "Invalid function argument"
- exit 1
- ;;
- esac
-
shift # shift past <MODE>
# The following loop is used to replace default light-node commandline parameters.
@@ -139,20 +148,26 @@ run_node() {
# cleanup data directory
if [ -z "$KEEP_DATA" ]; then
- rm -rf "$BOOTSTRAP_DIR" && mkdir "$BOOTSTRAP_DIR"
- rm -rf "$TEZOS_DIR" && mkdir "$TEZOS_DIR"
+ rm -rf $TEZOS_DIR $BOOTSTRAP_DIR || true
+ mkdir -p $TEZOS_DIR $BOOTSTRAP_DIR
fi
# protocol_runner needs 'libtezos.so' to run
export LD_LIBRARY_PATH="${BASH_SOURCE%/*}/tezos/interop/lib_tezos/artifacts:${BASH_SOURCE%/*}/target/$PROFILE"
# start node
- cargo run $CARGO_PROFILE_ARG --bin light-node -- \
+ if [ -z "$CARGO_BUILD_TARGET" ]; then
+ PROTOCOL_RUNNER_BINARY=./target/$PROFILE/protocol-runner
+ else
+ PROTOCOL_RUNNER_BINARY=./target/$CARGO_BUILD_TARGET/$PROFILE/protocol-runner
+ fi
+
+ cargo run --bin light-node -- \
--config-file "$CONFIG_FILE" \
--tezos-data-dir "$TEZOS_DIR" \
--identity-file "$IDENTITY_FILE" \
--bootstrap-db-path "$BOOTSTRAP_DIR" \
--network "$NETWORK" \
- --protocol-runner "./target/$PROFILE/protocol-runner" "${args[@]}"
+ --protocol-runner $PROTOCOL_RUNNER_BINARY "${args[@]}"
}
run_docker() {
@@ -174,32 +189,12 @@ run_sandbox() {
# -n | --network
NETWORK=sandbox
- # set compilation profile and cargo commandline argument
- PROFILE=""
- case $1 in
- debug)
- PROFILE="debug"
- shift
- ;;
- release)
- CARGO_PROFILE_ARG="--release"
- PROFILE="release"
- shift
- ;;
- *)
- echo "Invalid function argument"
- exit 1
- ;;
- esac
-
- shift # shift past <MODE>
-
# protocol_runner needs 'libtezos.so' to run
export LD_LIBRARY_PATH="${BASH_SOURCE%/*}/tezos/interop/lib_tezos/artifacts:${BASH_SOURCE%/*}/target/$PROFILE"
export TEZOS_CLIENT_UNSAFE_DISABLE_DISCLAIMER="Y"
- cargo run $CARGO_PROFILE_ARG --bin sandbox -- \
+ cargo run --bin sandbox -- \
--log-level "info" \
--sandbox-rpc-port "3030" \
--light-node-path "./target/$PROFILE/light-node" \
@@ -211,27 +206,38 @@ case $1 in
node)
warn_if_not_using_recommended_rust
- printf "\033[1;37mRunning Tezedge node in DEBUG mode\e[0m\n"
- build_all "debug"
- run_node "debug" "$@"
+ configure_env_variables --debug
+ print_configuration
+ build_all
+ run_node "$@"
+ ;;
+
+ node-saddr)
+ warn_if_not_using_recommended_rust
+ configure_env_variables --debug --addrsanitizer
+ print_configuration
+ build_all
+ run_node "$@"
;;
release)
warn_if_not_using_recommended_rust
- printf "\033[1;37mRunning Tezedge node in RELEASE mode\e[0m\n"
- build_all "release"
- run_node "release" "$@"
+ configure_env_variables --release
+ print_configuration
+ build_all
+ run_node "$@"
;;
sandbox)
warn_if_not_using_recommended_rust
- printf "\033[1;37mRunning Tezedge node in RELEASE mode\e[0m\n"
- build_all "release"
- run_sandbox "release" "$@"
+ configure_env_variables --release
+ print_configuration
+ build_all
+ run_sandbox "$@"
;;
docker)
- printf "\033[1;37mRunning Tezedge node in docker\e[0m\n"
+ echo -ne "\033[1;37mRunning Tezedge node in docker\e[0m\n"
run_docker "$@"
;;

diff:
@@ -14,7 +14,7 @@ getset = "0.1"
hex = "0.4"
itertools = "0.10"
num_cpus = "1.13"
-rocksdb = "0.15"
+rocksdb = {version = "0.15", features = ["snappy", "lz4", "zstd", "zlib"], default-features = false }
serde = { version = "1.0", features = ["derive", "rc"] }
slog = "2.7"
# local dependencies

diff:
@@ -10,7 +10,9 @@ enum-iterator = "0.6"
failure = "0.1"
chrono = { version = "0.4", features = ["serde"] }
lazy_static = "1.4"
-ocaml-interop = { version = "0.4.4", features = ["docs-rs"] }
+# patched version of v.0.4.4 - once new version of ocaml interop
+# is integrated git reference should be no longer used
+ocaml-interop = { git="https://github.com/simplestaking/ocaml-interop", branch="v0.4.4_saddr_fix"}
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
# local dependencies

diff:
@@ -9,7 +9,11 @@ failure = "0.1"
futures = { version = "0.3", features = ["thread-pool"] }
hex = "0.4"
lazy_static = "1.4"
-ocaml-interop = { version = "0.4.4", features = ["docs-rs"] }
+
+# patched version of v.0.4.4 - once new version of ocaml interop
+# is integrated git reference should be no longer used
+ocaml-interop = { git="https://github.com/simplestaking/ocaml-interop", branch="v0.4.4_saddr_fix"}
+
serde_json = "1.0"
# local dependencies
tezos_api = { path = "../api" }

diff:
@@ -8,6 +8,8 @@ edition = "2018"
crate-type = ["dylib", "rlib"]
[dependencies]
-ocaml-interop = { version = "0.4.4", features = ["docs-rs"] }
+# patched version of v.0.4.4 - once new version of ocaml interop
+# is integrated git reference should be no longer used
+ocaml-interop = { git="https://github.com/simplestaking/ocaml-interop/", branch="v0.4.4_saddr_fix"}
# local dependencies
tezos_context = { path = "../context" }


message: ocaml-interop dependency version bump

diff:
@@ -27,6 +27,7 @@ jobs:
echo '[build]' >> ~/.cargo/config
echo 'rustflags = ["-Z", "sanitizer=address"]' >> ~/.cargo/config
echo 'rustdocflags = ["-Z", "sanitizer=address"]' >> ~/.cargo/config
+ echo 'target = "x86_64-unknown-linux-gnu"' >> ~/.cargo/config
- name: Settings for cargo in OSX
if: runner.os == 'macOS'
run: |
@@ -47,15 +48,9 @@ jobs:
uses: actions-rs/cargo@v1
with:
command: check
- - name: cargo test SADDR
- if: matrix.sanitizer == 'address'
- uses: actions-rs/cargo@v1
- with:
- command: test
- name: cargo test
- if: matrix.sanitizer == 'none'
+ if: matrix.sanitizer == 'none' || matrix.os == 'ubuntu-latest'
uses: actions-rs/cargo@v1
with:
command: test
-

diff:
@@ -10,9 +10,7 @@ enum-iterator = "0.6"
failure = "0.1"
chrono = { version = "0.4", features = ["serde"] }
lazy_static = "1.4"
-# patched version of v.0.4.4 - once new version of ocaml interop
-# is integrated git reference should be no longer used
-ocaml-interop = { git="https://github.com/simplestaking/ocaml-interop", branch="v0.4.4_saddr_fix"}
+ocaml-interop = { version = "0.4.5", features = ["docs-rs"] }
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
# local dependencies

diff:
@@ -9,11 +9,7 @@ failure = "0.1"
futures = { version = "0.3", features = ["thread-pool"] }
hex = "0.4"
lazy_static = "1.4"
-
-# patched version of v.0.4.4 - once new version of ocaml interop
-# is integrated git reference should be no longer used
-ocaml-interop = { git="https://github.com/simplestaking/ocaml-interop", branch="v0.4.4_saddr_fix"}
-
+ocaml-interop = { version = "0.4.5", features = ["docs-rs"] }
serde_json = "1.0"
# local dependencies
tezos_api = { path = "../api" }

diff:
@@ -8,8 +8,6 @@ edition = "2018"
crate-type = ["dylib", "rlib"]
[dependencies]
-# patched version of v.0.4.4 - once new version of ocaml interop
-# is integrated git reference should be no longer used
-ocaml-interop = { git="https://github.com/simplestaking/ocaml-interop/", branch="v0.4.4_saddr_fix"}
+ocaml-interop = { version = "0.4.5", features = ["docs-rs"] }
# local dependencies
tezos_context = { path = "../context" }

original_message: ocaml-interop dependency version bump


message: temporary disable address sanitizer for tezos_messages package

diff:
@@ -49,8 +49,14 @@ jobs:
with:
command: check
- name: cargo test
- if: matrix.sanitizer == 'none' || matrix.os == 'ubuntu-latest'
+ if: matrix.sanitizer == 'none'
uses: actions-rs/cargo@v1
with:
command: test
+ - name: cargo test
+ if: matrix.sanitizer == 'address' && matrix.os == 'ubuntu-latest'
+ uses: actions-rs/cargo@v1
+ with:
+ command: test
+ args: --all --exclude tezos_messages

diff:
@@ -2011,8 +2011,9 @@ checksum = "8d3b63360ec3cb337817c2dbd47ab4a0f170d285d8e5a2064600f3def1402397"
[[package]]
name = "ocaml-interop"
-version = "0.4.4"
-source = "git+https://github.com/simplestaking/ocaml-interop?branch=v0.4.4_saddr_fix#511d74095c4ba95408707e4ed4db7d0be8a73ab4"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90d094d3c6bfbefc8ea5a0e4b3ba542105cbf1262fc44478e12cb23eee4391c1"
dependencies = [
"ocaml-sys",
]


diff:
// Copyright (c) SimpleStaking and Tezedge Contributors
// SPDX-License-Identifier: MIT
-use std::collections::HashMap;
use std::env;
use std::ffi::OsString;
use std::fs;
@@ -9,17 +8,27 @@ use std::io::{self, BufRead};
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::time::Duration;
+use std::{collections::HashMap, fmt::Debug};
use clap::{App, Arg};
+use rocksdb::ColumnFamilyDescriptor;
use shell::peer_manager::P2p;
use shell::PeerConnectionThreshold;
-use storage::persistent::{DbConfiguration, DbConfigurationBuilder};
+use storage;
+use storage::persistent::KeyValueSchema;
use tezos_api::environment;
use tezos_api::environment::TezosEnvironment;
use tezos_api::ffi::PatchContext;
use tezos_wrapper::TezosApiConnectionPoolConfiguration;
+const STORAGES_COUNT: usize = 3;
+const MINIMAL_THREAD_COUNT: usize = 1;
+const DB_STORAGE_VERSION: i64 = 16;
+const DB_CONTEXT_STORAGE_VERSION: i64 = 16;
+const DB_CONTEXT_ACTIONS_STORAGE_VERSION: i64 = 16;
+const LRU_CACHE_SIZE_32MB: usize = 32 * 1024 * 1024;
+
#[derive(Debug, Clone)]
pub struct Rpc {
pub listener_port: u16,
@@ -34,9 +43,71 @@ pub struct Logging {
pub file: Option<PathBuf>,
}
+pub trait ColumnFactory {
+ fn create(&self, cache: &rocksdb::Cache) -> Vec<ColumnFamilyDescriptor>;
+}
+#[derive(Debug, Clone)]
+pub struct DBTableInitializer {}
+
+#[derive(Debug, Clone)]
+pub struct ContextTableInitializer {}
+
+#[derive(Debug, Clone)]
+pub struct ContextActionsTableInitializer {}
+
+impl ColumnFactory for DBTableInitializer {
+ fn create(&self, cache: &rocksdb::Cache) -> Vec<ColumnFamilyDescriptor> {
+ vec![
+ storage::block_storage::BlockPrimaryIndex::descriptor(cache),
+ storage::block_storage::BlockByLevelIndex::descriptor(cache),
+ storage::block_storage::BlockByContextHashIndex::descriptor(cache),
+ storage::BlockMetaStorage::descriptor(cache),
+ storage::OperationsStorage::descriptor(cache),
+ storage::OperationsMetaStorage::descriptor(cache),
+ storage::SystemStorage::descriptor(cache),
+ storage::persistent::sequence::Sequences::descriptor(cache),
+ storage::MempoolStorage::descriptor(cache),
+ storage::ChainMetaStorage::descriptor(cache),
+ storage::PredecessorStorage::descriptor(cache),
+ ]
+ }
+}
+
+impl ColumnFactory for ContextTableInitializer {
+ fn create(&self, cache: &rocksdb::Cache) -> Vec<ColumnFamilyDescriptor> {
+ vec![
+ storage::SystemStorage::descriptor(cache),
+ storage::merkle_storage::MerkleStorage::descriptor(cache),
+ ]
+ }
+}
+
+impl ColumnFactory for ContextActionsTableInitializer {
+ fn create(&self, cache: &rocksdb::Cache) -> Vec<ColumnFamilyDescriptor> {
+ vec![
+ storage::SystemStorage::descriptor(cache),
+ storage::context_action_storage::ContextActionByBlockHashIndex::descriptor(cache),
+ storage::context_action_storage::ContextActionByContractIndex::descriptor(cache),
+ storage::context_action_storage::ContextActionByTypeIndex::descriptor(cache),
+ storage::context_action_storage::ContextActionStorage::descriptor(cache),
+ ]
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct RocksDBConfig<C: ColumnFactory> {
+ pub cache_size: usize,
+ pub expected_db_version: i64,
+ pub db_path: PathBuf,
+ pub columns: C,
+ pub threads: Option<usize>,
+}
+
#[derive(Debug, Clone)]
pub struct Storage {
- pub db_cfg: DbConfiguration,
+ pub db: RocksDBConfig<DBTableInitializer>,
+ pub db_context: RocksDBConfig<ContextTableInitializer>,
+ pub db_context_actions: RocksDBConfig<ContextActionsTableInitializer>,
pub db_path: PathBuf,
pub tezos_data_dir: PathBuf,
pub store_context_actions: bool,
@@ -689,28 +760,48 @@ impl Environment {
}
},
},
- storage: crate::configuration::Storage {
- tezos_data_dir: data_dir.clone(),
- db_cfg: {
- let mut db_cfg = DbConfigurationBuilder::default();
-
- if let Some(value) = args.value_of("db-cfg-max-threads") {
- let max_treads = value
- .parse::<usize>()
- .expect("Provided value cannot be converted to number");
- db_cfg.max_threads(Some(max_treads));
- }
-
- db_cfg.build().unwrap()
- },
- db_path: {
- let db_path = args
+ storage: {
+ let path = args
.value_of("bootstrap-db-path")
.unwrap_or("")
.parse::<PathBuf>()
.expect("Provided value cannot be converted to path");
- get_final_path(&data_dir, db_path)
- },
+ let db_path = get_final_path(&data_dir, path);
+
+ let threads_count = args.value_of("db-cfg-max-threads").map(|value| {
+ value
+ .parse::<usize>()
+ .map(|val| std::cmp::min(MINIMAL_THREAD_COUNT, val / STORAGES_COUNT))
+ .expect("Provided value cannot be converted to number")
+ });
+
+ let db = RocksDBConfig {
+ cache_size: LRU_CACHE_SIZE_32MB,
+ expected_db_version: DB_STORAGE_VERSION,
+ db_path: db_path.clone().join("db"),
+ columns: DBTableInitializer {},
+ threads: threads_count,
+ };
+ let db_context = RocksDBConfig {
+ cache_size: LRU_CACHE_SIZE_32MB,
+ expected_db_version: DB_CONTEXT_STORAGE_VERSION,
+ db_path: db_path.join("context"),
+ columns: ContextTableInitializer {},
+ threads: threads_count,
+ };
+ let db_context_actions = RocksDBConfig {
+ cache_size: LRU_CACHE_SIZE_32MB,
+ expected_db_version: DB_CONTEXT_ACTIONS_STORAGE_VERSION,
+ db_path: db_path.join("context_actions"),
+ columns: ContextActionsTableInitializer {},
+ threads: threads_count,
+ };
+ crate::configuration::Storage {
+ tezos_data_dir: data_dir.clone(),
+ db,
+ db_context,
+ db_context_actions,
+ db_path: db_path,
store_context_actions: args
.value_of("store-context-actions")
.unwrap_or("true")
@@ -728,8 +819,9 @@ impl Environment {
// validate valid json
if let Err(e) = serde_json::from_str::<
HashMap<String, serde_json::Value>,
- >(&content)
- {
+ >(
+ &content
+ ) {
panic!(
"Invalid json file: {}, reason: {}",
path.as_path().display().to_string(),
@@ -756,6 +848,7 @@ impl Environment {
}
}
},
+ }
},
identity: crate::configuration::Identity {
identity_json_file_path: {

diff:
@@ -6,8 +6,9 @@ use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;
+use configuration::{ColumnFactory, RocksDBConfig};
use riker::actors::*;
-use rocksdb::Cache;
+use rocksdb::{Cache, DB};
use slog::{crit, debug, error, info, o, warn, Drain, Logger};
use logging::detailed_json;
@@ -23,15 +24,11 @@ use shell::mempool::init_mempool_state_storage;
use shell::mempool::mempool_prevalidator::MempoolPrevalidator;
use shell::peer_manager::PeerManager;
use shell::shell_channel::{ShellChannel, ShellChannelTopic, ShuttingDown};
-use storage::context::TezedgeContext;
-use storage::merkle_storage::MerkleStorage;
-use storage::persistent::sequence::Sequences;
-use storage::persistent::{open_cl, open_kv, CommitLogSchema, KeyValueSchema, PersistentStorage};
+use storage::persistent::DbConfiguration;
+use storage::persistent::{open_cl, open_kv, CommitLogSchema, PersistentStorage};
use storage::{
- block_storage, check_database_compatibility, context_action_storage,
- resolve_storage_init_chain_data, BlockMetaStorage, BlockStorage, ChainMetaStorage,
- ContextActionStorage, MempoolStorage, OperationsMetaStorage, OperationsStorage,
- PredecessorStorage, StorageInitInfo, SystemStorage,
+ check_database_compatibility, context::TezedgeContext, persistent::DBError,
+ resolve_storage_init_chain_data, BlockStorage, StorageInitInfo,
};
use tezos_api::environment;
use tezos_api::environment::TezosEnvironmentConfiguration;
@@ -467,6 +464,36 @@ fn block_on_actors(
});
}
+fn initialize_db<Factory: ColumnFactory>(
+ log: &Logger,
+ cache: &Cache,
+ config: &RocksDBConfig<Factory>,
+ env: &TezosEnvironmentConfiguration,
+) -> Result<Arc<DB>, DBError> {
+ let columns: Vec<_> = config.columns.create(cache);
+ let db = open_kv(
+ &config.db_path,
+ columns,
+ &DbConfiguration {
+ max_threads: config.threads,
+ },
+ )
+ .map(Arc::new)?;
+
+ match check_database_compatibility(db.clone(), config.expected_db_version, &env, &log) {
+ Ok(false) => Err(DBError::DatabaseIncompatibility {
+ name: format!(
+ "Database is incompatible with version {}",
+ config.expected_db_version
+ ),
+ }),
+ Err(e) => Err(DBError::DatabaseIncompatibility {
+ name: format!("Failed to verify database compatibility reason: '{}'", e),
+ }),
+ _ => Ok(db),
+ }
+}
+
fn main() {
// Parses config + cli args
let env = crate::configuration::Environment::from_args();
@@ -516,60 +543,49 @@ fn main() {
// create common RocksDB block cache to be shared among column families
// IMPORTANT: Cache object must live at least as long as DB (returned by open_kv)
- let cache = Cache::new_lru_cache(128 * 1024 * 1024).unwrap(); // 128 MB
-
- let schemas = vec![
- block_storage::BlockPrimaryIndex::descriptor(&cache),
- block_storage::BlockByLevelIndex::descriptor(&cache),
- block_storage::BlockByContextHashIndex::descriptor(&cache),
- BlockMetaStorage::descriptor(&cache),
- OperationsStorage::descriptor(&cache),
- OperationsMetaStorage::descriptor(&cache),
- context_action_storage::ContextActionByBlockHashIndex::descriptor(&cache),
- context_action_storage::ContextActionByContractIndex::descriptor(&cache),
- context_action_storage::ContextActionByTypeIndex::descriptor(&cache),
- ContextActionStorage::descriptor(&cache),
- MerkleStorage::descriptor(&cache),
- SystemStorage::descriptor(&cache),
- Sequences::descriptor(&cache),
- MempoolStorage::descriptor(&cache),
- ChainMetaStorage::descriptor(&cache),
- PredecessorStorage::descriptor(&cache),
+ let cache = [
+ Cache::new_lru_cache(env.storage.db.cache_size).unwrap(),
+ Cache::new_lru_cache(env.storage.db_context.cache_size).unwrap(),
+ Cache::new_lru_cache(env.storage.db_context_actions.cache_size).unwrap(),
];
- let rocks_db = match open_kv(&env.storage.db_path, schemas, &env.storage.db_cfg) {
- Ok(db) => Arc::new(db),
- Err(e) => shutdown_and_exit!(
- error!(log, "Failed to create RocksDB database at '{:?}'", &env.storage.db_path; "reason" => e),
- actor_system
- ),
- };
- debug!(log, "Loaded RocksDB database");
+ let storages: Result<Vec<Arc<DB>>, DBError> = vec![
+ initialize_db(&log, &cache[0], &env.storage.db, &tezos_env),
+ initialize_db(&log, &cache[1], &env.storage.db_context, &tezos_env),
+ initialize_db(&log, &cache[2], &env.storage.db_context_actions, &tezos_env),
+ ]
+ .into_iter()
+ .collect();
- match check_database_compatibility(rocks_db.clone(), DATABASE_VERSION, &tezos_env, &log) {
- Ok(false) => shutdown_and_exit!(
- crit!(log, "Database incompatibility detected"),
- actor_system
- ),
+ let storages = match storages {
Err(e) => shutdown_and_exit!(
- error!(log, "Failed to verify database compatibility"; "reason" => e),
+ crit!(
+ log,
+ "Failed to create initialize RocksDB databases '{:?}'",
+ e
+ ),
actor_system
),
- _ => (),
- }
-
- let schemas = vec![BlockStorage::descriptor()];
+ Ok(dbs) => dbs,
+ };
{
- let commit_logs = match open_cl(&env.storage.db_path, schemas) {
+ let commit_logs = match open_cl(&env.storage.db_path, vec![BlockStorage::descriptor()]) {
Ok(commit_logs) => Arc::new(commit_logs),
Err(e) => shutdown_and_exit!(
error!(log, "Failed to open commit logs"; "reason" => e),
actor_system
),
};
-
- let persistent_storage = PersistentStorage::new(rocks_db, commit_logs);
+ if let [kv, kv_context, kv_actions] = &storages[..] {
+ debug!(log, "Loaded RocksDB databases");
+
+ let persistent_storage = PersistentStorage::new(
+ kv.clone(),
+ kv_context.clone(),
+ kv_actions.clone(),
+ commit_logs,
+ );
let tezedge_context = TezedgeContext::new(
BlockStorage::new(&persistent_storage),
persistent_storage.merkle(),
@@ -596,5 +612,7 @@ fn main() {
actor_system
),
}
+ } else {
+ }
}
}

diff:
@@ -12,13 +12,13 @@ use slog::{warn, Logger};
use crypto::hash::{BlockHash, ChainId, HashType};
use tezos_messages::p2p::encoding::block_header::Level;
-use crate::num_from_slice;
use crate::persistent::database::{IteratorMode, IteratorWithSchema};
use crate::persistent::{
default_table_options, Decoder, Encoder, KeyValueSchema, KeyValueStoreWithSchema,
PersistentStorage, SchemaError,
};
use crate::predecessor_storage::{PredecessorKey, PredecessorStorage};
+use crate::{num_from_slice, persistent::StorageType};
use crate::{BlockHeaderWithHash, StorageError};
pub type BlockMetaStorageKV = dyn KeyValueStoreWithSchema<BlockMetaStorage> + Sync + Send;
@@ -52,7 +52,7 @@ impl BlockMetaStorage {
pub fn new(persistent_storage: &PersistentStorage) -> Self {
BlockMetaStorage {
- kv: persistent_storage.kv(),
+ kv: persistent_storage.kv(StorageType::Database),
predecessors_index: PredecessorStorage::new(persistent_storage),
}
}

diff:
@@ -13,7 +13,7 @@ use crypto::hash::{
use crate::persistent::{
BincodeEncoded, CommitLogSchema, CommitLogWithSchema, KeyValueSchema, KeyValueStoreWithSchema,
- Location, PersistentStorage,
+ Location, PersistentStorage, StorageType,
};
use crate::{BlockHeaderWithHash, Direction, IteratorMode, StorageError};
@@ -125,9 +125,11 @@ pub trait BlockStorageReader: Sync + Send {
impl BlockStorage {
pub fn new(persistent_storage: &PersistentStorage) -> Self {
Self {
- primary_index: BlockPrimaryIndex::new(persistent_storage.kv()),
- by_level_index: BlockByLevelIndex::new(persistent_storage.kv()),
- by_context_hash_index: BlockByContextHashIndex::new(persistent_storage.kv()),
+ primary_index: BlockPrimaryIndex::new(persistent_storage.kv(StorageType::Database)),
+ by_level_index: BlockByLevelIndex::new(persistent_storage.kv(StorageType::Database)),
+ by_context_hash_index: BlockByContextHashIndex::new(
+ persistent_storage.kv(StorageType::Database),
+ ),
clog: persistent_storage.clog(),
}
}

diff:
@@ -13,7 +13,7 @@ use tezos_messages::Head;
use crate::persistent::{
default_table_options, BincodeEncoded, Decoder, Encoder, KeyValueSchema,
- KeyValueStoreWithSchema, PersistentStorage, SchemaError,
+ KeyValueStoreWithSchema, PersistentStorage, SchemaError, StorageType,
};
use crate::StorageError;
@@ -59,7 +59,7 @@ pub struct ChainMetaStorage {
impl ChainMetaStorage {
pub fn new(persistent_storage: &PersistentStorage) -> Self {
Self {
- kv: persistent_storage.kv(),
+ kv: persistent_storage.kv(StorageType::Database),
}
}

diff:
@@ -18,7 +18,6 @@ use crypto::hash::{
use tezos_context::channel::ContextAction;
use tezos_messages::base::signature_public_key_hash::{ConversionError, SignaturePublicKeyHash};
-use crate::num_from_slice;
use crate::persistent::codec::range_from_idx_len;
use crate::persistent::sequence::{SequenceGenerator, SequenceNumber};
use crate::persistent::{
@@ -26,6 +25,7 @@ use crate::persistent::{
KeyValueStoreWithSchema, PersistentStorage, SchemaError,
};
use crate::StorageError;
+use crate::{num_from_slice, persistent::StorageType};
pub enum ContextHashType {
Block,
@@ -76,12 +76,13 @@ pub struct ContextActionStorage {
impl ContextActionStorage {
pub fn new(persistent_storage: &PersistentStorage) -> Self {
+ let storage = persistent_storage.kv(StorageType::ContextAction);
Self {
- kv: persistent_storage.kv(),
+ kv: persistent_storage.kv(StorageType::ContextAction),
generator: persistent_storage.seq().generator(Self::name()),
- context_by_block_index: ContextActionByBlockHashIndex::new(persistent_storage.kv()),
- context_by_contract_index: ContextActionByContractIndex::new(persistent_storage.kv()),
- context_by_type_index: ContextActionByTypeIndex::new(persistent_storage.kv()),
+ context_by_block_index: ContextActionByBlockHashIndex::new(storage.clone()),
+ context_by_contract_index: ContextActionByContractIndex::new(storage.clone()),
+ context_by_type_index: ContextActionByTypeIndex::new(storage),
}
}

diff:
@@ -493,7 +493,7 @@ pub mod tests_common {
let cache = Cache::new_lru_cache(128 * 1024 * 1024)?; // 128 MB
let kv = open_kv(
- &path,
+ path.join("db"),
vec![
block_storage::BlockPrimaryIndex::descriptor(&cache),
block_storage::BlockByLevelIndex::descriptor(&cache),
@@ -501,26 +501,43 @@ pub mod tests_common {
BlockMetaStorage::descriptor(&cache),
OperationsStorage::descriptor(&cache),
OperationsMetaStorage::descriptor(&cache),
- context_action_storage::ContextActionByBlockHashIndex::descriptor(&cache),
- context_action_storage::ContextActionByContractIndex::descriptor(&cache),
- context_action_storage::ContextActionByTypeIndex::descriptor(&cache),
- MerkleStorage::descriptor(&cache),
SystemStorage::descriptor(&cache),
Sequences::descriptor(&cache),
DatabaseBackedSkipList::descriptor(&cache),
Lane::descriptor(&cache),
ListValue::descriptor(&cache),
MempoolStorage::descriptor(&cache),
- ContextActionStorage::descriptor(&cache),
ChainMetaStorage::descriptor(&cache),
PredecessorStorage::descriptor(&cache),
],
&cfg,
)?;
+
+ let kv_context = open_kv(
+ path.join("context"),
+ vec![MerkleStorage::descriptor(&cache)],
+ &cfg,
+ )?;
+
+ let kv_context_action = open_kv(
+ path.join("context_actions"),
+ vec![
+ ContextActionStorage::descriptor(&cache),
+ context_action_storage::ContextActionByBlockHashIndex::descriptor(&cache),
+ context_action_storage::ContextActionByContractIndex::descriptor(&cache),
+ context_action_storage::ContextActionByTypeIndex::descriptor(&cache),
+ ],
+ &cfg,
+ )?;
let clog = open_cl(&path, vec![BlockStorage::descriptor()])?;
Ok(Self {
- persistent_storage: PersistentStorage::new(Arc::new(kv), Arc::new(clog)),
+ persistent_storage: PersistentStorage::new(
+ Arc::new(kv),
+ Arc::new(kv_context),
+ Arc::new(kv_context_action),
+ Arc::new(clog),
+ ),
path,
remove_on_destroy,
})

diff:
@@ -15,7 +15,7 @@ use tezos_messages::p2p::encoding::operation::OperationMessage;
use crate::persistent::{
BincodeEncoded, Decoder, Encoder, KeyValueSchema, KeyValueStoreWithSchema, PersistentStorage,
- SchemaError,
+ SchemaError, StorageType,
};
use crate::{num_from_slice, IteratorMode, StorageError};
@@ -65,7 +65,7 @@ pub struct MempoolStorage {
impl MempoolStorage {
pub fn new(persistent_storage: &PersistentStorage) -> Self {
Self {
- kv: persistent_storage.kv(),
+ kv: persistent_storage.kv(StorageType::Database),
}
}

diff:
@@ -10,12 +10,12 @@ use rocksdb::{Cache, ColumnFamilyDescriptor, MergeOperands};
use crypto::hash::{BlockHash, ChainId, HashType};
use tezos_messages::p2p::encoding::prelude::*;
-use crate::num_from_slice;
use crate::persistent::database::{IteratorMode, IteratorWithSchema};
use crate::persistent::{
default_table_options, Decoder, Encoder, KeyValueSchema, KeyValueStoreWithSchema,
PersistentStorage, SchemaError,
};
+use crate::{num_from_slice, persistent::StorageType};
use crate::{BlockHeaderWithHash, StorageError};
/// Convenience type for operation meta storage database
@@ -30,7 +30,7 @@ pub struct OperationsMetaStorage {
impl OperationsMetaStorage {
pub fn new(persistent_storage: &PersistentStorage) -> Self {
Self {
- kv: persistent_storage.kv(),
+ kv: persistent_storage.kv(StorageType::Database),
}
}

diff:
@@ -12,7 +12,7 @@ use tezos_messages::p2p::encoding::prelude::*;
use crate::persistent::{
default_table_options, Decoder, Encoder, KeyValueSchema, KeyValueStoreWithSchema,
- PersistentStorage, SchemaError,
+ PersistentStorage, SchemaError, StorageType,
};
use crate::StorageError;
@@ -35,7 +35,7 @@ pub struct OperationsStorage {
impl OperationsStorage {
pub fn new(persistent_storage: &PersistentStorage) -> Self {
Self {
- kv: persistent_storage.kv(),
+ kv: persistent_storage.kv(StorageType::Database),
}
}

diff:
@@ -27,6 +27,8 @@ pub enum DBError {
RocksDBError { error: Error },
#[fail(display = "Column family {} is missing", name)]
MissingColumnFamily { name: &'static str },
+ #[fail(display = "Database incompatibility {}", name)]
+ DatabaseIncompatibility { name: String },
}
impl From<SchemaError> for DBError {

diff:
@@ -26,7 +26,7 @@ pub mod sequence;
#[derive(Builder, Debug, Clone)]
pub struct DbConfiguration {
#[builder(default = "None")]
- max_threads: Option<usize>,
+ pub max_threads: Option<usize>,
}
impl Default for DbConfiguration {
@@ -116,7 +116,12 @@ where
#[derive(Clone)]
pub struct PersistentStorage {
/// key-value store
+ //TODO to be renamed to kv_db
kv: Arc<DB>,
+ /// context store
+ kv_context: Arc<DB>,
+ /// context actions store
+ kv_context_actions: Arc<DB>,
/// commit log store
clog: Arc<CommitLogs>,
/// autoincrement id generators
@@ -125,20 +130,37 @@ pub struct PersistentStorage {
merkle: Arc<RwLock<MerkleStorage>>,
}
+pub enum StorageType {
+ Database,
+ Context,
+ ContextAction,
+}
+
impl PersistentStorage {
- pub fn new(kv: Arc<DB>, clog: Arc<CommitLogs>) -> Self {
+ pub fn new(
+ kv: Arc<DB>,
+ kv_context: Arc<DB>,
+ kv_context_actions: Arc<DB>,
+ clog: Arc<CommitLogs>,
+ ) -> Self {
let seq = Arc::new(Sequences::new(kv.clone(), 1000));
Self {
clog,
- kv: kv.clone(),
+ kv,
+ kv_context: kv_context.clone(),
+ kv_context_actions,
seq,
- merkle: Arc::new(RwLock::new(MerkleStorage::new(kv))),
+ merkle: Arc::new(RwLock::new(MerkleStorage::new(kv_context))),
}
}
#[inline]
- pub fn kv(&self) -> Arc<DB> {
- self.kv.clone()
+ pub fn kv(&self, storage: StorageType) -> Arc<DB> {
+ match storage {
+ StorageType::Context => self.kv_context.clone(),
+ StorageType::ContextAction => self.kv_context_actions.clone(),
+ StorageType::Database => self.kv.clone(),
+ }
}
#[inline]

diff:
@@ -8,13 +8,13 @@ use serde::{Deserialize, Serialize};
use crypto::hash::BlockHash;
-use crate::block_meta_storage::Meta;
use crate::persistent::database::{IteratorMode, IteratorWithSchema};
use crate::persistent::{
default_table_options, BincodeEncoded, KeyValueSchema, KeyValueStoreWithSchema,
PersistentStorage,
};
use crate::StorageError;
+use crate::{block_meta_storage::Meta, persistent::StorageType};
pub type PredecessorsIndexStorageKV = dyn KeyValueStoreWithSchema<PredecessorStorage> + Sync + Send;
@@ -41,7 +41,7 @@ pub struct PredecessorStorage {
impl PredecessorStorage {
pub fn new(persistent_storage: &PersistentStorage) -> Self {
Self {
- kv: persistent_storage.kv(),
+ kv: persistent_storage.kv(StorageType::Database),
}
}

diff:
@@ -71,6 +71,11 @@ fn context_get_values_by_block_hash() -> Result<(), Error> {
storage.put_action(&block_hash_2, value_2_0)?;
storage.put_action(&block_hash_1, value_1_1)?;
storage.put_action(&block_hash_2, value_2_1)?;
+ tmp_storage
+ .storage()
+ .kv(persistent::StorageType::Context)
+ .flush()
+ .unwrap();
// block hash 1
let values = storage.get_by_block_hash(&block_hash_1)?;

diff:
use maplit::hashmap;
-use storage::skip_list::{Lane, SkipListError, TryExtend, TypedLane};
use storage::tests_common::TmpStorage;
+use storage::{
+ persistent::StorageType,
+ skip_list::{Lane, SkipListError, TryExtend, TypedLane},
+};
#[test]
fn lane_new() {
@@ -12,8 +15,8 @@ fn lane_new() {
let lane = Lane::new(
1,
0,
- tmp_storage.storage().kv(),
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage.storage().seq().generator("__lane:lane_new"),
);
assert_eq!(lane.level(), 0);
@@ -25,8 +28,8 @@ fn lane_higher() {
let lane = Lane::new(
2,
0,
- tmp_storage.storage().kv(),
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage.storage().seq().generator("__lane:lane_higher"),
);
let higher_lane = lane.higher_lane();
@@ -39,8 +42,8 @@ fn lane_lower() {
let lane = Lane::new(
3,
1,
- tmp_storage.storage().kv(),
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage.storage().seq().generator("__lane:lane_lower"),
);
let lower_lane = lane.lower_lane();
@@ -53,8 +56,8 @@ fn lane_lower_underflow() {
let lane = Lane::new(
4,
0,
- tmp_storage.storage().kv(),
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()
@@ -70,8 +73,8 @@ fn lane_put_get_values() {
let mut lane = Lane::new(
5,
0,
- tmp_storage.storage().kv(),
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()

diff:
@@ -11,7 +11,7 @@ use rand::{
};
use serde::{Deserialize, Serialize};
-use storage::persistent::BincodeEncoded;
+use storage::persistent::{BincodeEncoded, StorageType};
use storage::skip_list::{DatabaseBackedSkipList, TypedSkipList};
use storage::tests_common::TmpStorage;
@@ -41,7 +41,7 @@ fn list_new() {
let list: Box<dyn TypedSkipList<i32, i32>> = Box::new(
DatabaseBackedSkipList::new(
1,
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()
@@ -58,7 +58,7 @@ fn list_push() {
let mut list: Box<dyn TypedSkipList<i32, i32>> = Box::new(
DatabaseBackedSkipList::new(
2,
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()
@@ -88,7 +88,7 @@ fn list_check_first() {
let mut list: Box<dyn TypedSkipList<i32, i32>> = Box::new(
DatabaseBackedSkipList::new(
3,
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()
@@ -114,7 +114,7 @@ fn list_check_second() {
let mut list: Box<dyn TypedSkipList<i32, i32>> = Box::new(
DatabaseBackedSkipList::new(
4,
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()
@@ -143,7 +143,7 @@ fn list_check_bottom_lane() {
let mut list: Box<dyn TypedSkipList<i32, i32>> = Box::new(
DatabaseBackedSkipList::new(
5,
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()
@@ -173,7 +173,7 @@ pub fn list_check_faster_lane() {
let mut list: Box<dyn TypedSkipList<i32, i32>> = Box::new(
DatabaseBackedSkipList::new(
6,
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()
@@ -203,7 +203,7 @@ pub fn list_check_lane_traversal() {
let mut list: Box<dyn TypedSkipList<i32, i32>> = Box::new(
DatabaseBackedSkipList::new(
7,
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()
@@ -233,7 +233,7 @@ pub fn list_get_value_by_key() {
let mut list: Box<dyn TypedSkipList<i32, i32>> = Box::new(
DatabaseBackedSkipList::new(
8,
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()
@@ -265,7 +265,7 @@ pub fn list_get_values_by_prefix() -> Result<(), failure::Error> {
let mut list: Box<dyn TypedSkipList<String, i32>> = Box::new(
DatabaseBackedSkipList::new(
9,
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()
@@ -323,7 +323,7 @@ pub fn list_check_lane_order_traversal() {
let mut list: Box<dyn TypedSkipList<i32, i32>> = Box::new(
DatabaseBackedSkipList::new(
8,
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()
@@ -358,7 +358,7 @@ pub fn list_check_get_key() {
let mut list: Box<dyn TypedSkipList<i32, i32>> = Box::new(
DatabaseBackedSkipList::new(
8,
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()
@@ -386,7 +386,7 @@ pub fn skip_list_simulate_ledger() {
let list: Box<dyn TypedSkipList<u64, Operation>> = Box::new(
DatabaseBackedSkipList::new(
8,
- tmp_storage.storage().kv(),
+ tmp_storage.storage().kv(StorageType::Database),
tmp_storage
.storage()
.seq()

original_message:
[TE-280] dedicated database instances for context and context actions

Signed-off-by: Mateusz Nowakowski <mateusz.nowakowski@simplestaking.com>