idx,query,code,docstring
1,priority queue," def push(self, item, priority=None):
priority = item if priority is None else priority
node = PriorityQueueNode(item, priority)
for index, current in enumerate(self.priority_queue_list):
if current.priority < node.priority:
self.priority_queue_list.insert(index, node)
return
self.priority_queue_list.append(node)","Push the item in the priority queue.
if priority is not given, priority is set to the value of item.
when traversed complete queue"
2,custom http error response," def handle_http_error(self, response, custom_messages=None,
raise_for_status=False):
if not custom_messages:
custom_messages = {}
if response.status_code in custom_messages.keys():
raise errors.HTTPError(custom_messages[response.status_code])
if raise_for_status:
response.raise_for_status()","Converts service errors to Python exceptions
Parameters
----------
response : requests.Response
A service response.
custom_messages : dict, optional
A mapping of custom exception messages to HTTP status codes.
raise_for_status : bool, optional
If True, the requests library provides Python exceptions.
Returns
-------
None"
3,get executable path,"def which(name):
for p in os.environ['PATH'].split(os.pathsep):
exe = os.path.join(p, name)
if is_executable(exe):
return os.path.abspath(exe)
for ext in [''] + os.environ.get('PATHEXT', '').split(os.pathsep):
exe = '{}{}'.format(exe, ext.lower())
if is_executable(exe):
return os.path.abspath(exe)",
4,parse query string in url,"def strip_url_params3(url, strip=None):
if not strip: strip = []
parse = urllib.parse.urlparse(url)
query = urllib.parse.parse_qs(parse.query)
query = {k: v[0] for k, v in query.items() if k not in strip}
query = urllib.parse.urlencode(query)
new = parse._replace(query=query)
return new.geturl()",
5,custom http error response," def handle_http_error(self, response, custom_messages=None,
raise_for_status=False):
if not custom_messages:
custom_messages = {}
if response.status_code in custom_messages.keys():
raise errors.HTTPError(custom_messages[response.status_code])
if raise_for_status:
response.raise_for_status()","Converts service errors to Python exceptions
Parameters
----------
response : requests.Response
A service response.
custom_messages : dict, optional
A mapping of custom exception messages to HTTP status codes.
raise_for_status : bool, optional
If True, the requests library provides Python exceptions.
Returns
-------
None"
6,k means clustering,"def cluster_kmeans(data, n_clusters, **kwargs):
km = cl.KMeans(n_clusters, **kwargs)
kmf = km.fit(data)
labels = kmf.labels_
return labels, [np.nan]","Identify clusters using K - Means algorithm.
Parameters
----------
data : array_like
array of size [n_samples, n_features].
n_clusters : int
The number of clusters expected in the data.
Returns
-------
dict
boolean array for each identified cluster."
7,export to excel," def to_excel(self, *args):
path = os.getcwd()
fname = self.fname.replace("".ppl"", ""_ppl"") + "".xlsx""
if len(args) > 0 and args[0] != """":
path = args[0]
if os.path.exists(path) == False:
os.mkdir(path)
xl_file = pd.ExcelWriter(path + os.sep + fname)
for idx in self.filter_data(""""):
self.extract(idx)
labels = list(self.filter_data("""").values())
for prof in self.data:
data_df = pd.DataFrame()
data_df[""X""] = self.data[prof][0]
for timestep, data in zip(self.time, self.data[prof][1]):
data_df[timestep] = data
myvar = labels[prof-1].split("" "")[0]
br_label = labels[prof-1].split(""\'"")[5]
unit = labels[prof-1].split(""\'"")[7].replace(""/"", ""-"")
mylabel = ""{} - {} - {}"".format(myvar, br_label, unit)
data_df.to_excel(xl_file, sheet_name=mylabel)
xl_file.save()
","Dump all the data to excel, fname and path can be passed as args"
8,aes encryption,"def aes_encrypt(base64_encryption_key, data):
if isinstance(data, text_type):
data = data.encode(""UTF-8"")
aes_key_bytes, hmac_key_bytes = _extract_keys(base64_encryption_key)
data = _pad(data)
iv_bytes = os.urandom(AES_BLOCK_SIZE)
cipher = AES.new(aes_key_bytes, mode=AES.MODE_CBC, IV=iv_bytes)
data = iv_bytes + cipher.encrypt(data)
hmac_signature = hmac.new(hmac_key_bytes, data, hashlib.sha256).digest()
return as_base64(data + hmac_signature)","Encrypt data with AES-CBC and sign it with HMAC-SHA256
Arguments:
base64_encryption_key (str): a base64-encoded string containing an AES encryption key
and HMAC signing key as generated by generate_encryption_key()
data (str): a byte string containing the data to be encrypted
Returns:
str: the encrypted data as a byte string with the HMAC signature appended to the end
prepend init vector"
9,how to get current date,"def date_0utc(date):
return ee.Date.fromYMD(date.get('year'), date.get('month'),
date.get('day'))","Get the 0 UTC date for a date
Parameters
----------
date : ee.Date
Returns
-------
ee.Date"
10,export to excel," def save(self):
pyexcel_export.save_data(self.excel_filename, data=self.excel_raw, retain_meta=True,
created=self.created, modified=datetime.now().isoformat())",
11,heatmap from 3d coordinates,"def heatmap(z, x=None, y=None, colorscale='Viridis'):
z = np.atleast_1d(z)
data = [go.Heatmap(z=z, x=x, y=y, colorscale=colorscale)]
return Chart(data=data)","Create a heatmap.
Parameters
----------
z : TODO
x : TODO, optional
y : TODO, optional
colorscale : TODO, optional
Returns
-------
Chart"
12,parse json file,"def _read_json_file(path):
try:
with open(path) as f:
return json.load(f)
except:
print ('Warning: Unable to read or parse json file: '+path)
return None",
13,sending binary data over a serial connection," def write(self, data):
try:
if isinstance(data, str) or (sys.version_info < (3,) and isinstance(data, unicode)):
data = data.encode('utf-8')
self._device.write(data)
except serial.SerialTimeoutException:
pass
except serial.SerialException as err:
raise CommError('Error writing to device.', err)
else:
self.on_write(data=data)","Writes data to the device.
:param data: data to write
:type data: string
:raises: py:class:`~alarmdecoder.util.CommError`
Hack to support unicode under Python 2.x"
14,scatter plot," def scatter_plot(self, ax, topic_dims, t=None, ms_limits=True, **kwargs_plot):
plot_specs = {'marker': 'o', 'linestyle': 'None'}
plot_specs.update(kwargs_plot)
data = self.data_t(topic_dims, t)
ax.plot(*(data.T), **plot_specs)
if ms_limits:
ax.axis(self.axes_limits(topic_dims))","2D or 3D scatter plot.
:param axes ax: matplotlib axes (use Axes3D if 3D data)
:param tuple topic_dims: list of (topic, dims) tuples, where topic is a string and dims is a list of dimensions to be plotted for that topic.
:param int t: time indexes to be plotted
:param dict kwargs_plot: argument to be passed to matplotlib's plot function, e.g. the style of the plotted points 'or'
:param bool ms_limits: if set to True, automatically set axes boundaries to the sensorimotor boundaries (default: True)
t_bound = float('inf')
if t is None:
for topic, _ in topic_dims:
t_bound = min(t_bound, self.counts[topic])
t = range(t_bound)
data = self.pack(topic_dims, t)"
15,get the description of a http status code,"def error_code_to_str(code):
try:
name = errno.errorcode[code]
except KeyError:
name = ""UNKNOWN""
try:
description = os.strerror(code)
except ValueError:
description = ""no description available""
return ""{} (errno {}): {}"".format(name, code, description)","Converts a given error code (errno) to a useful and human readable string.
:param int code: a possibly invalid/unknown error code
:rtype: str
:returns: a string explaining and containing the given error code, or a string
explaining that the errorcode is unknown if that is the case"
16,write csv,"def write_csv(filename, data, delimiter=CSV_DELIMITER):
with open(filename, 'w') as file:
csv_writer = csv.writer(file, delimiter=delimiter)
for line in data:
csv_writer.writerow(line)","Write image data to CSV file
:param filename: name of CSV file to write data to
:type filename: str
:param data: image data to write to CSV file
:type data: numpy array
:param delimiter: delimiter used in CSV file. Default is ``;``
:type delimiter: str"
17,linear regression," def linear_regression(self, target, regression_length, mask=NotSpecified):
from .statistical import RollingLinearRegression
return RollingLinearRegression(
dependent=self,
independent=target,
regression_length=regression_length,
mask=mask,
)","Construct a new Factor that performs an ordinary least-squares
regression predicting the columns of `self` from `target`.
This method can only be called on factors which are deemed safe for use
as inputs to other factors. This includes `Returns` and any factors
created from `Factor.rank` or `Factor.zscore`.
Parameters
----------
target : zipline.pipeline.Term with a numeric dtype
The term to use as the predictor/independent variable in each
regression. This may be a Factor, a BoundColumn or a Slice. If
`target` is two-dimensional, regressions are computed asset-wise.
regression_length : int
Length of the lookback window over which to compute each
regression.
mask : zipline.pipeline.Filter, optional
A Filter describing which assets should be regressed with the
target slice each day.
Returns
-------
regressions : zipline.pipeline.factors.RollingLinearRegression
A new Factor that will compute linear regressions of `target`
against the columns of `self`.
Examples
--------
Suppose we want to create a factor that regresses AAPL's 10-day returns
against the 10-day returns of all other assets, computing each
regression over 30 days. This can be achieved by doing the following::
returns = Returns(window_length=10)
returns_slice = returns[sid(24)]
aapl_regressions = returns.linear_regression(
target=returns_slice, regression_length=30,
)
This is equivalent to doing::
aapl_regressions = RollingLinearRegressionOfReturns(
target=sid(24), returns_length=10, regression_length=30,
)
See Also
--------
:func:`scipy.stats.linregress`
:class:`zipline.pipeline.factors.RollingLinearRegressionOfReturns`"
18,encrypt aes ctr mode," def encrypt(cls, data, key, iv_data):
validate_key_size(key, cls.key_size, ""AES"")
iv, ctr = iv_data
ciphertext = Crypto_AES.new(key, Crypto_AES.MODE_CTR,
counter=ctr).encrypt(data)
return iv + ciphertext",
19,export to excel,"def export_to_xlsx(table, filename_or_fobj=None, sheet_name=""Sheet1"", *args, **kwargs):
workbook = Workbook()
sheet = workbook.active
sheet.title = sheet_name
prepared_table = prepare_to_export(table, *args, **kwargs)
field_names = next(prepared_table)
for col_index, field_name in enumerate(field_names):
cell = sheet.cell(row=1, column=col_index + 1)
cell.value = field_name
_convert_row = _python_to_cell(list(map(table.fields.get, field_names)))
for row_index, row in enumerate(prepared_table, start=1):
for col_index, (value, number_format) in enumerate(_convert_row(row)):
cell = sheet.cell(row=row_index + 1, column=col_index + 1)
cell.value = value
if number_format is not None:
cell.number_format = number_format
return_result = False
if filename_or_fobj is None:
filename_or_fobj = BytesIO()
return_result = True
source = Source.from_file(filename_or_fobj, mode=""wb"", plugin_name=""xlsx"")
workbook.save(source.fobj)
source.fobj.flush()
if return_result:
source.fobj.seek(0)
result = source.fobj.read()
else:
result = source.fobj
if source.should_close:
source.fobj.close()
return result","Export the rows.Table to XLSX file and return the saved file.
Write header
Write sheet rows"
20,copy to clipboard,"def copy_to_clipboard(i):
s=i['string']
failed=False
ee=''
try:
import pyperclip
except Exception as e:
ee=format(e)
failed=True
pass
if not failed:
pyperclip.copy(s)
else:
failed=False
try:
from Tkinter import Tk
except ImportError as e:
ee=format(e)
failed=True
pass
if failed:
failed=False
try:
from tkinter import Tk
except ImportError as e:
ee=format(e)
failed=True
pass
if failed:
return {'return':1, 'error':'none of pyperclip/Tkinter/tkinter packages is installed'}
try:
r = Tk()
r.withdraw()
r.clipboard_clear()
r.clipboard_append(s)
r.destroy()
except Exception as e:
return {'return':1, 'error':'problem copying string to clipboard ('+format(e)+')'}
return {'return':0}","pragma: no cover
Input: {
string - string to copy
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
Try to load pyperclip (seems to work fine on Windows)
Try to load Tkinter
Copy to clipboard"
21,get executable path,"def get_compiler_path():
given_binary = os.environ.get('SOLC_BINARY')
if given_binary:
return given_binary
for path in os.getenv('PATH', '').split(os.pathsep):
path = path.strip('""')
executable_path = os.path.join(path, BINARY)
if os.path.isfile(executable_path) and os.access(
executable_path, os.X_OK):
return executable_path
return None","Return the path to the solc compiler.
This funtion will search for the solc binary in the $PATH and return the
path of the first executable occurence.
If the user provides a specific solc binary let's use that"
22,converting uint8 array to image,"def _convert_uint8(im):
if im.dtype != np.uint8:
im = np.uint8(im * 255)
return im",
23,how to check if a checkbox is checked," def checkbox_check(self, force_check=False):
if not self.get_attribute('checked'):
self.click(force_click=force_check)",Wrapper to check a checkbox
24,copy to clipboard,"def copy_to_clipboard(i):
s=i['string']
failed=False
ee=''
try:
import pyperclip
except Exception as e:
ee=format(e)
failed=True
pass
if not failed:
pyperclip.copy(s)
else:
failed=False
try:
from Tkinter import Tk
except ImportError as e:
ee=format(e)
failed=True
pass
if failed:
failed=False
try:
from tkinter import Tk
except ImportError as e:
ee=format(e)
failed=True
pass
if failed:
return {'return':1, 'error':'none of pyperclip/Tkinter/tkinter packages is installed'}
try:
r = Tk()
r.withdraw()
r.clipboard_clear()
r.clipboard_append(s)
r.destroy()
except Exception as e:
return {'return':1, 'error':'problem copying string to clipboard ('+format(e)+')'}
return {'return':0}","pragma: no cover
Input: {
string - string to copy
Output: {
return - return code = 0, if successful
> 0, if error
(error) - error text if return > 0
}
Try to load pyperclip (seems to work fine on Windows)
Try to load Tkinter
Copy to clipboard"
25,memoize to disk - persistent memoization,"def memoize(obj):
cache = obj.cache = {}
@functools.wraps(obj)
def memoizer(*args, **kwargs):
key = tuple(list(args) + sorted(kwargs.items()))
if key not in cache:
cache[key] = obj(*args, **kwargs)
return cache[key]
return memoizer",
26,get current process id,"Error: 404 {""message"": ""Not Found"", ""documentation_url"": ""https://docs.github.com/rest/repos/repos","get-a-repository"", ""status"": ""404""}"
27,positions of substrings in string,"def substitute_globals(string, globs=None):
sub = set(re.findall('\{(.*?)\}', string))
globs = globs or inspect.currentframe().f_back.f_globals
if sub:
for item in map(str, sub):
string = string.replace(""${%s}""%item, globs[item])
return string
else:
return False",
28,linear regression,"def func(X, y):
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import cross_val_score
model = LinearRegression()
model.fit(X, y)
return model.predict(X)",
29,binomial distribution,"def EvalBinomialPmf(k, n, p):
return scipy.stats.binom.pmf(k, n, p)","Evaluates the binomial pmf.
Returns the probabily of k successes in n trials with probability p."
30,unzipping large files," def unzip(self, in_file, out_file):
with ZipFile(in_file) as zf:
zf.extract('collection.anki2', path=self.tempdir)
shutil.move(os.path.join(self.tempdir, 'collection.anki2'),
out_file)
return out_file",
31,copying a file to a path,"def _copy(from_path, to_path):
message = ""Copying path [%s] to [%s]""
log.debug(message, from_path, to_path)
copy(from_path, to_path)",
32,copying a file to a path," def _copy_file(self, file_obj, destination, suffix, overwrite):
if overwrite:
raise NotImplementedError
filename = self._generate_new_filename(file_obj.file.name, suffix)
file_obj.pk = None
file_obj.id = None
file_obj.save()
file_obj.folder = destination
file_obj._file_data_changed_hint = False
file_obj.file = file_obj._copy_file(filename)
file_obj.original_filename = self._generate_new_filename(file_obj.original_filename, suffix)
file_obj.save()","Not yet implemented as we have to find a portable (for different storage backends) way to overwrite files
We are assuming here that we are operating on an already saved database objects with current database state available
Due to how inheritance works, we have to set both pk and id to None
no need to update size, sha1, etc."
33,parse json file," def __parse_json_file(self, file_path):
if file_path == '' or os.path.splitext(file_path)[1] != '.json':
raise IOError('Invalid Json file')
with open(file_path) as json_file:
self._raw_data = json.load(json_file)
self._json_data = copy.deepcopy(self._raw_data)","Process Json file data
:@param file_path
:@type file_path: string
:@throws IOError"
34,copying a file to a path," def write_file_to_output(self, filename, path):
path = os.path.join(self.out_path, path)
if self.add_filehash_to_path and os.path.exists(path):
return
path_part = os.path.dirname(path)
PathDumper.__makedirs(path_part)
shutil.copy(filename, path)
os.chmod(path, 0o666)
return path",Avoid rewriting existing files
35,httpclient post json," async def _http_post(self, url, data):
data = json.dumps(data)
headers = {""Authorization"": ""GoogleLogin auth={0}"".format(self.token),
""Content-type"": ""application/json""}
res = await self.session.request(
'POST',
FULL_SJ_URL + url,
data=data,
headers=headers,
params={'tier': 'aa',
'hl': 'en_US',
'dv': 0,
'alt': 'json'})
ret = await res.json()
return ret",
36,encrypt aes ctr mode,"def _encrypt(data):
BS = AES.block_size
def pad(s):
n = BS - len(s) % BS
char = chr(n).encode('utf8')
return s + n * char
password = settings.GECKOBOARD_PASSWORD
salt = Random.new().read(BS - len('Salted__'))
key, iv = _derive_key_and_iv(password, salt, 32, BS)
cipher = AES.new(key, AES.MODE_CBC, iv)
encrypted = b'Salted__' + salt + cipher.encrypt(pad(data))
return base64.b64encode(encrypted)",Equivalent to OpenSSL using 256 bit AES in CBC mode
37,connect to sql," def connect(self):
self.close()
self._connect = pymysql.connect(**self._db_options)
self._connect.autocommit(True)",
38,how to check if a checkbox is checked,"def assert_checked_checkbox(self, value):
check_box = find_field(world.browser, 'checkbox', value)
assert check_box, ""Cannot find checkbox '{}'."".format(value)
assert check_box.is_selected(), ""Check box should be selected.""","Assert the checkbox with label (recommended), name or id is checked."
39,sending binary data over a serial connection," def send(self, msg):
slipDriver = sliplib.Driver()
slipData = slipDriver.send(msg)
res = self._serialPort.write(slipData)
return res","Encodes data to slip protocol and then sends over serial port
Uses the SlipLib module to convert the message data into SLIP format.
The message is then sent over the serial port opened with the instance
of the Faraday class used when invoking send().
Args:
msg (bytes): Bytes format message to send over serial port.
Returns:
int: Number of bytes transmitted over the serial port.
Create a sliplib Driver
Package data in slip format
Send data over serial port
Return number of bytes transmitted over serial port"
40,convert a date string into yyyymmdd,"def date_string_to_date(p_date):
result = None
if p_date:
parsed_date = re.match(r'(\d{4})-(\d{2})-(\d{2})', p_date)
if parsed_date:
result = date(
int(parsed_date.group(1)),
int(parsed_date.group(2)),
int(parsed_date.group(3))
)
else:
raise ValueError
return result","Given a date in YYYY-MM-DD, returns a Python date object. Throws a
ValueError if the date is invalid.
year
month
day"
41,how to read .csv file in an efficient way?," def read_csv(self, file_path, use_whole_file=False, names=None, skiprows=0,
*args, **kwargs):
def csv_file(partition_number, files):
file_count = 0
for _, contents in files:
if partition_number == 0 and file_count == 0 and _skiprows > 0:
yield pandas.read_csv(
sio(contents), *args,
header=None,
names=mynames,
skiprows=_skiprows,
**kwargs)
else:
file_count += 1
yield pandas.read_csv(
sio(contents), *args,
header=None,
names=mynames,
**kwargs)
def csv_rows(partition_number, rows):
in_str = ""\n"".join(rows)
if partition_number == 0:
return iter([
pandas.read_csv(
sio(in_str), *args, header=None,
names=mynames,
skiprows=_skiprows,
**kwargs)])
else:
return iter([pandas.read_csv(sio(in_str), *args, header=None,
names=mynames, **kwargs)])
mynames = None
_skiprows = skiprows
if names:
mynames = names
else:
first_line = self.spark_ctx.textFile(file_path).first()
frame = pandas.read_csv(sio(first_line), **kwargs)
mynames = list(frame.columns)
_skiprows += 1
if use_whole_file:
return self.from_pandas_rdd(
self.spark_ctx.wholeTextFiles(file_path)
.mapPartitionsWithIndex(csv_file))
else:
return self.from_pandas_rdd(
self.spark_ctx.textFile(file_path)
.mapPartitionsWithIndex(csv_rows))","Read a CSV file in and parse it into Pandas DataFrames. By default,
the first row from the first partition of that data is parsed and used
as the column names for the data from. If no 'names' param is
provided we parse the first row of the first partition of data and
use it for column names.
Parameters
----------
file_path: string
Path to input. Any valid file path in Spark works here, eg:
'file:///my/path/in/local/file/system' or 'hdfs:/user/juliet/'
use_whole_file: boolean
Whether of not to use the whole file.
names: list of strings, optional
skiprows: integer, optional
indicates how many rows of input to skip. This will
only be applied to the first partition of the data (so if
#skiprows > #row in first partition this will not work). Generally
this shouldn't be an issue for small values of skiprows.
No other value of header is supported.
All additional parameters available in pandas.read_csv() are usable
here.
Returns
-------
A SparklingPandas DataFrame that contains the data from the
specified file.
pylint: disable=unexpected-keyword-arg
Only skip lines on the first file
pylint: disable=unexpected-keyword-arg
could use .iterows instead?
If we need to peak at the first partition and determine the column
names
In the future we could avoid this expensive call.
pylint sees frame as a tuple despite it being a DataFrame
Do the actual load"
42,string to date," def to_string(self):
return '-'.join([d.strftime('%Y-%m-%d') for d in (self.date_a, self.date_b)])",
43,positions of substrings in string,"def get_substring_idxs(substr, string):
return [match.start() for match in re.finditer(substr, string)]","Return a list of indexes of substr. If substr not found, list is
empty.
Arguments:
substr (str): Substring to match.
string (str): String to match in.
Returns:
list of int: Start indices of substr."
44,create cookie," def setcookie(self, key, value, max_age=None, expires=None, path='/', domain=None, secure=None, httponly=False):
newcookie = Morsel()
newcookie.key = key
newcookie.value = value
newcookie.coded_value = value
if max_age is not None:
newcookie['max-age'] = max_age
if expires is not None:
newcookie['expires'] = expires
if path is not None:
newcookie['path'] = path
if domain is not None:
newcookie['domain'] = domain
if secure:
newcookie['secure'] = secure
if httponly:
newcookie['httponly'] = httponly
self.sent_cookies = [c for c in self.sent_cookies if c.key != key]
self.sent_cookies.append(newcookie)",Add a new cookie
45,how to make the checkbox checked,"def check_checkbox(self, value):
check_box = find_field(world.browser, 'checkbox', value)
assert check_box, ""Cannot find checkbox '{}'."".format(value)
if not check_box.is_selected():
check_box.click()","Check the checkbox with label (recommended), name or id."
46,pretty print json,"def json_pretty_print(d, file=None):
args = {'sort_keys': True, 'indent': 2, 'separators': (',', ': ')}
if file:
return json.dump(d, file, **args)
return json.dumps(d, **args)",
47,aes encryption,"def aes_encrypt(value, secret, block_size=AES.block_size):
iv = os.urandom(block_size * 2)
cipher = AES.new(secret[:32], AES.MODE_CFB, iv[:block_size])
return b'%s%s' % (iv, cipher.encrypt(value))","AES encrypt @value with @secret using the |CFB| mode of AES
with a cryptographically secure initialization vector.
-> (#bytes) AES encrypted @value
..
from vital.security import aes_encrypt, aes_decrypt
aes_encrypt(""Hello, world"",
""aLWEFlwgwlreWELFNWEFWLEgwklgbweLKWEBGW"")
# -> 'zYgVYMbeOuiHR50aMFinY9JsfyMQCvpzI+LNqNcmZhw='
aes_decrypt(
""zYgVYMbeOuiHR50aMFinY9JsfyMQCvpzI+LNqNcmZhw="",
""aLWEFlwgwlreWELFNWEFWLEgwklgbweLKWEBGW"")
# -> 'Hello, world'
..f"
48,how to get html of website," def get_html(self):
r = requests.get(self.url)
if r.status_code == 404:
raise PlayerNotFoundError
else:
soup = BeautifulSoup(r.text, 'html.parser')
return soup.find(""div"", class_=""pnl490M"")",
49,k means clustering,"def optimal_clustering(df, patch, method='kmeans', statistic='gap', max_K=5):
if len(patch) == 1:
return [patch]
if statistic == 'db':
if method == 'kmeans':
if len(patch) <= 5:
K_max = 2
else:
K_max = min(len(patch) / 2, max_K)
clustering = {}
db_index = []
X = df.ix[patch, :]
for k in range(2, K_max + 1):
kmeans = cluster.KMeans(n_clusters=k).fit(X)
clustering[k] = pd.DataFrame(kmeans.predict(X), index=patch)
dist_mu = squareform(pdist(kmeans.cluster_centers_))
sigma = []
for i in range(k):
points_in_cluster = clustering[k][clustering[k][0] == i].index
sigma.append(sqrt(X.ix[points_in_cluster, :].var(axis=0).sum()))
db_index.append(davies_bouldin(dist_mu, np.array(sigma)))
db_index = np.array(db_index)
k_optimal = np.argmin(db_index) + 2
return [list(clustering[k_optimal][clustering[k_optimal][0] == i].index) for i in range(k_optimal)]
elif method == 'agglomerative':
if len(patch) <= 5:
K_max = 2
else:
K_max = min(len(patch) / 2, max_K)
clustering = {}
db_index = []
X = df.ix[patch, :]
for k in range(2, K_max + 1):
agglomerative = cluster.AgglomerativeClustering(n_clusters=k, linkage='average').fit(X)
clustering[k] = pd.DataFrame(agglomerative.fit_predict(X), index=patch)
tmp = [list(clustering[k][clustering[k][0] == i].index) for i in range(k)]
centers = np.array([np.mean(X.ix[c, :], axis=0) for c in tmp])
dist_mu = squareform(pdist(centers))
sigma = []
for i in range(k):
points_in_cluster = clustering[k][clustering[k][0] == i].index
sigma.append(sqrt(X.ix[points_in_cluster, :].var(axis=0).sum()))
db_index.append(davies_bouldin(dist_mu, np.array(sigma)))
db_index = np.array(db_index)
k_optimal = np.argmin(db_index) + 2
return [list(clustering[k_optimal][clustering[k_optimal][0] == i].index) for i in range(k_optimal)]
elif statistic == 'gap':
X = np.array(df.ix[patch, :])
if method == 'kmeans':
f = cluster.KMeans
gaps = gap(X, ks=range(1, min(max_K, len(patch))), method=f)
k_optimal = list(gaps).index(max(gaps))+1
clustering = pd.DataFrame(f(n_clusters=k_optimal).fit_predict(X), index=patch)
return [list(clustering[clustering[0] == i].index) for i in range(k_optimal)]
else:
raise 'error: only db and gat statistics are supported'",
50,unzipping large files,"def unzip(zip_content, dest_dir):
with zipfile.ZipFile(zip_content, ""r"") as zf:
for member in zf.infolist():
words = member.filename.split('/')
path = dest_dir
for word in words[:-1]:
drive, word = os.path.splitdrive(word)
head, word = os.path.split(word)
if word in (os.curdir, os.pardir, ''):
continue
path = os.path.join(path, word)
zf.extract(member, path)","From http://stackoverflow.com/a/12886818
Path traversal defense copied from
http://hg.python.org/cpython/file/tip/Lib/http/server.py#l789"
51,scatter plot,"Error: 404 {""message"": ""No commit found for the ref 50d3f979e79e63c66629065c75595696dc79802e"", ""documentation_url"": ""https://docs.github.com/v3/repos/contents/"", ""status"": ""404""}",
52,matrix multiply," def __mul__(self, other):
if isinstance(other, Matrix):
return Matrix(self.matrix.dot(other.matrix))
else:
return Matrix(self.matrix * other)",
53,write csv,"def write_to_csv(fname, header, rows):
with open(fname, 'wb') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='|',
quoting=csv.QUOTE_MINIMAL)
csvwriter.writerow(header)
for row in rows:
csvwriter.writerow(
[s.encode(""utf-8"").replace(',', '').replace('\n', '')
for s in row])",
54,heatmap from 3d coordinates,"def colorize(img, heatmap):
heatmap = viz.intensity_to_rgb(heatmap, cmap='jet')[:, :, ::-1]
return img * 0.5 + heatmap * 0.5","img: bgr, [0,255]
heatmap: [0,1]"
55,parse command line argument,"def parse_command_line_arguments():
parser = argparse.ArgumentParser( description='Rotates the cell lattice in VASP POSCAR files' )
parser.add_argument( 'poscar', help=""filename of the VASP POSCAR to be processed"" )
parser.add_argument( '-a', '--axis', nargs=3, type=float, help=""vector for rotation axis"", required=True )
parser.add_argument( '-d', '--degrees', type=int, help=""rotation angle in degrees"", required=True )
args = parser.parse_args()
return( args )",
56,regex case insensitive," def match(self, string):
if self.casesensitive:
return self.pattern == os.path.normcase(string)
else:
return self.pattern.lower() == os.path.normcase(string).lower()",Returns True if the argument matches the constant.
57,how to make the checkbox checked," def checkbox_check(self, force_check=False):
if not self.get_attribute('checked'):
self.click(force_click=force_check)",Wrapper to check a checkbox
58,string similarity levenshtein,"def levenshtein(left, right):
sc = SparkContext._active_spark_context
jc = sc._jvm.functions.levenshtein(_to_java_column(left), _to_java_column(right))
return Column(jc)","Computes the Levenshtein distance of the two given strings.
>>> df0 = spark.createDataFrame([('kitten', 'sitting',)], ['l', 'r'])
>>> df0.select(levenshtein('l', 'r').alias('d')).collect()
[Row(d=3)]"
59,how to read .csv file in an efficient way?,"def read_csv(filename, has_header=True):
with open(filename) as fh:
csv_reader = csv.reader(fh)
header = None
if has_header:
header = csv_reader.next()
rows = [row for row in csv_reader]
return header, rows",
60,string to date," def to_string(self):
return '-'.join([d.strftime('%Y-%m-%d') for d in (self.date_a, self.date_b)])",
61,aes encryption," def encrypt_message(self, signed_message, message, key, iv):
raw = signed_message + message
block_size = AES.block_size
pad = lambda s: s + (block_size - len(s) % block_size) * chr(block_size - len(s) % block_size).encode('utf-8')
message_to_encrypt = pad(raw)
cipher = AES.new(key, AES.MODE_CBC, iv)
return cipher.encrypt(message_to_encrypt)",
62,how to get html of website," def get_html_source(self):
req = urllib.request.Request(self.url)
req.add_header(""user-agent"", random.choice(USER_AGENTS))
req_text = urllib.request.urlopen(req).read()
self.source = str(req_text)
self.soup = BeautifulSoup(self.source, ""html.parser"")
return self.source","Gets source page of url
:return: HTML source"
63,socket recv timeout," def recvall(self, timeout=0.5):
response = ''
self.socket.setblocking(False)
start = time.time()
while True:
if response and time.time() - start > timeout:
break
elif time.time() - start > timeout * 2:
break
try:
data = self.socket.recv(4096)
if data:
response += data.replace(self._rconreplystring, '')
start = time.time()
else:
time.sleep(0.1)
except socket.error:
pass
return response.strip()","Receive the RCON command response
:param timeout: The timeout between consequent data receive
:return str: The RCON command response with header stripped out"
64,get inner html," def innerHTML(self, html: str) -> None:
df = self._parse_html(html)
if self.connected:
self._set_inner_html_web(df.html)
self._empty()
self._append_child(df)","type: ignore
Set innerHTML both on this node and related browser node."
65,unique elements,"def unique(list):
unique = []; [unique.append(x) for x in list if x not in unique]
return unique ",Returns a copy of the list without duplicates.
66,get all parents of xml node," def get_edges(self, node):
return [
(node, child)
for child in self.get_children(node)
] + [
(parent, node)
for parent in self.get_parents(node)
]",
67,confusion matrix,"Error: 404 {""message"": ""Not Found"", ""documentation_url"": ""https://docs.github.com/rest/repos/repos","get-a-repository"", ""status"": ""404""}"
68,pretty print json,"def pprint_json(json_raw):
print(json.dumps(json.loads(json_raw), indent=2, sort_keys=True))",
69,how to randomly pick a number,"def randbelow(num: int) -> int:
if not isinstance(num, int):
raise TypeError('number must be an integer')
if num <= 0:
raise ValueError('number must be greater than zero')
if num == 1:
return 0
nbits = num.bit_length()
randnum = random_randint(nbits)
while randnum >= num:
randnum = random_randint(nbits)
return randnum","Return a random int in the range [0,num).
Raises ValueError if num <= 0, and TypeError if it's not an integer.
>>> randbelow(16) #doctest:+SKIP
13
https://github.com/python/cpython/blob/3.6/Lib/random.py#L223
don't use (n-1) here because n can be 1
0 <= randnum < 2**nbits"
70,convert json to csv,"def json_to_csv(json_input):
try:
json_input = json.loads(json_input)
except:
pass
headers = set()
for json_row in json_input:
headers.update(json_row.keys())
csv_io = StringIO.StringIO()
csv_out = csv.DictWriter(csv_io,headers)
csv_out.writeheader()
for json_row in json_input:
csv_out.writerow(json_row)
csv_io.seek(0)
return csv_io.read()","Convert simple JSON to CSV
Accepts a JSON string or JSON object
If loads fails, it's probably already parsed"
71,connect to sql," def __mysql_connect(self, connect_using_database_name=True):
try:
conn = self.__mysql_driver.connect(host=self.__mysql_host, port=self.__mysql_port, user=self.__mysql_user, passwd=self.__mysql_passwd)
conn.set_character_set(self.__mysql_encoding)
if connect_using_database_name:
conn.select_db(self.__mysql_db)
return conn
except Exception as e:
raise Exception(""could not connect to database: %s"" % e)",
72,encode url," def _urlencode(self, h):
rv = []
for k,v in h.iteritems():
rv.append('%s=%s' %
(urllib.quote(k.encode(""utf-8"")),
urllib.quote(v.encode(""utf-8""))))
return '&'.join(rv)",
73,k means clustering," def kmeans_clustering(self, numc, X=None, npcs=15):
from sklearn.cluster import KMeans
if X is None:
D_sub = self.adata.uns['X_processed']
X = (
D_sub -
D_sub.mean(0)).dot(
self.adata.uns['pca_obj'].components_[
:npcs,
:].T)
save = True
else:
save = False
cl = KMeans(n_clusters=numc).fit_predict(Normalizer().fit_transform(X))
if save:
self.adata.obs['kmeans_clusters'] = pd.Categorical(cl)
else:
return cl","Performs k-means clustering.
Parameters
----------
numc - int
Number of clusters
npcs - int, optional, default 15
Number of principal components to use as inpute for k-means
clustering."
74,aes encryption," def _cbc_encrypt(self, content, final_key):
aes = AES.new(final_key, AES.MODE_CBC, self._enc_iv)
padding = (16 - len(content) % AES.block_size)
for _ in range(padding):
content += chr(padding).encode()
temp = bytes(content)
return aes.encrypt(temp)",This method encrypts the content.
75,parse query string in url,"def _parse_url(url):
p = urlsplit(url)
query = {k: v[0] for k, v in parse_qs(p.query).items() if len(v) == 1}
return ''.join([p.netloc, p.path]), query",
76,get executable path,"def get_binary_path(executable, logging_level='INFO'):
if sys.platform == 'win32':
if executable == 'start':
return executable
executable = executable + '.exe'
if executable in os.listdir('.'):
binary = os.path.join(os.getcwd(), executable)
else:
binary = next((os.path.join(path, executable)
for path in os.environ['PATH'].split(os.pathsep)
if os.path.isfile(os.path.join(path, executable))), None)
else:
venv_parent = get_venv_parent_path()
venv_bin_path = os.path.join(venv_parent, '.venv', 'bin')
if not venv_bin_path in os.environ.get('PATH'):
if logging_level == 'DEBUG':
print(f'Adding path {venv_bin_path} to environment PATH variable')
os.environ['PATH'] = os.pathsep.join([os.environ['PATH'], venv_bin_path])
binary = shutil.which(executable)
return binary if binary else None",Gets the software name and returns the path of the binary.
77,get current process id,"Error: 404 {""message"": ""Not Found"", ""documentation_url"": ""https://docs.github.com/rest/repos/repos","get-a-repository"", ""status"": ""404""}"
78,html encode string," def _code_no_lexer(self, text):
text = text.encode(charset).strip()
return(
% houdini.escape_html(text)
)","encode to utf8 string
"
79,httpclient post json," def post(self, json=None):
return self._call('post', url=self.endpoint, json=json)","Send a POST request and return the JSON decoded result.
Args:
json (dict, optional): Object to encode and send in request.
Returns:
mixed: JSON decoded response data."
80,convert int to string,"def string_to_int( s ):
result = 0
for c in s:
if not isinstance(c, int): c = ord( c )
result = 256 * result + c
return result","Convert a string of bytes into an integer, as per X9.62."
81,copy to clipboard,"def copy_to_clipboard(text):
if sys.platform == 'darwin':
os.system('echo ""{0}"" | pbcopy'.format(text))
return
try:
from Tkinter import Tk
except ImportError:
return
r = Tk()
r.withdraw()
r.clipboard_clear()
r.clipboard_append(text.encode('ascii'))
r.destroy()","reliable on mac
okay we'll try cross-platform way"
82,encode url,"def url_encode(url):
if isinstance(url, text_type):
url = url.encode('utf8')
return quote(url, ':/%?&=')","Convert special characters using %xx escape.
:param url: str
:return: str - encoded url"
83,linear regression," def linreg(self, rsq=False, conf_test=False):
model='linear'
if not self.datestack:
self.compute_dt_stats()
if np.isnan(self.min_dt_ptp):
max_dt_ptp = calcperc(self.dt_stack_ptp, (4, 96))[1]
self.min_dt_ptp = 0.20 * max_dt_ptp
if self.robust:
model='theilsen'
print(""Compute stack linear trend with model: %s"" % model)
self.stack_trend, self.stack_intercept, self.stack_detrended_std = \
ma_linreg(self.ma_stack, self.date_list, dt_stack_ptp=self.dt_stack_ptp, min_dt_ptp=self.min_dt_ptp, \
n_thresh=self.n_thresh, model=model, rsq=False, conf_test=False, smooth=False, n_cpu=self.n_cpu)","This could fail if stack contains a small number of inputs
model='ransac'"
84,how to extract zip file recursively,"def extract_zip(zip_name, exclude_term=None):
zip_dir = os.path.dirname(os.path.abspath(zip_name))
try:
with zipfile.ZipFile(zip_name) as z:
files = [zip_file for zip_file in z.namelist() if not zip_file.endswith('/')]
print('Extracting %i files from %r.' % (len(files), zip_name))
for zip_file in files:
if exclude_term:
dest_file = zip_file.replace(exclude_term, '')
else:
dest_file = zip_file
dest_file = os.path.normpath(os.path.join(zip_dir, dest_file))
dest_dir = os.path.dirname(dest_file)
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
data = z.read(zip_file)
with open(dest_file, 'wb') as f:
f.write(encode_utf8(data))
except zipfile.error as e:
print(""Bad zipfile (%r): %s"" % (zip_name, e))
raise e","Extracts a zip file to its containing directory.
write each zipped file out if it isn't a directory
remove any provided extra directory term from zip file
make directory if it does not exist
read file from zip, then write to new directory"
85,how to extract zip file recursively,"def extract_zip(archive_path, dest):
with zipfile.ZipFile(archive_path) as z:
validate_filenames(z.namelist())
z.extractall(dest)
for info in z.filelist:
if not info.filename.endswith('/'):
mode = (info.external_attr >> 16) & 0o777
if mode & stat.S_IXUSR:
os.chmod(os.path.join(dest, info.filename), 0o755)","Set file permissions. Tar does this by default, but with zip we need
to do it ourselves.
This is how to get file permissions out of a zip archive,
according to http://stackoverflow.com/q/434641/823869 and
http://bugs.python.org/file34873/issue15795_cleaned.patch.
Don't copy the whole mode, just set the executable bit. Two
reasons for this. 1) This is all going to end up in a git
tree, which only records the executable bit anyway. 2) Zip's
support for Unix file modes is nonstandard, so the mode field
is often zero and could be garbage. Mistakenly setting a file
executable isn't a big deal, but e.g. removing read
permissions would cause an error."
86,convert int to string,"def string_to_int( s ):
result = 0
for c in s:
if not isinstance(c, int): c = ord( c )
result = 256 * result + c
return result","Convert a string of bytes into an integer, as per X9.62."
87,matrix multiply," def __mul__(self, other):
if isinstance(other, Matrix):
return Matrix(self.matrix.dot(other.matrix))
else:
return Matrix(self.matrix * other)",
88,connect to sql," def connect(self, db_uri, debug=False):
kwargs = {'echo': debug, 'convert_unicode': True}
if 'mysql' in db_uri:
kwargs['pool_recycle'] = 3600
elif '://' not in db_uri:
logger.debug(""detected sqlite path URI: {}"".format(db_uri))
db_path = os.path.abspath(os.path.expanduser(db_uri))
db_uri = ""sqlite:///{}"".format(db_path)
self.engine = create_engine(db_uri, **kwargs)
logger.debug('connection established successfully')
BASE.metadata.bind = self.engine
self.session = scoped_session(sessionmaker(bind=self.engine))
self.query = self.session.query
return self","Configure connection to a SQL database.
Args:
db_uri (str): path/URI to the database to connect to
debug (Optional[bool]): whether to output logging information
connect to the SQL database
make sure the same engine is propagated to the BASE classes
start a session
shortcut to query method"
89,fuzzy match ranking," def fuzzmatch(self, fuzzkey, multi=False):
keys, ratios = np.array([(f, seqm(None, fuzzkey, f).ratio()) for f in self.components.keys()]).T
mratio = max(ratios)
if multi:
return keys[ratios == mratio]
else:
if sum(ratios == mratio) == 1:
return keys[ratios == mratio][0]
else:
raise ValueError(""\nThe filter key provided ('{:}') matches two or more filter names equally well:\n"".format(fuzzkey) + ', '.join(keys[ratios == mratio]) + ""\nPlease be more specific!"")","Identify a filter by fuzzy string matching.
Partial ('fuzzy') matching performed by `fuzzywuzzy.fuzzy.ratio`
Parameters
----------
fuzzkey : str
A string that partially matches one filter name more than the others.
Returns
-------
The name of the most closely matched filter. : str"
90,convert int to bool,"def convert_to_bool(x: Any, default: bool = None) -> bool:
if isinstance(x, bool):
return x
if not x:
return default
try:
return int(x) != 0
except (TypeError, ValueError):
pass
try:
return float(x) != 0
except (TypeError, ValueError):
pass
if not isinstance(x, str):
raise Exception(""Unknown thing being converted to bool: {!r}"".format(x))
x = x.upper()
if x in [""Y"", ""YES"", ""T"", ""TRUE""]:
return True
if x in [""N"", ""NO"", ""F"", ""FALSE""]:
return False
raise Exception(""Unknown thing being converted to bool: {!r}"".format(x))","Transforms its input to a ``bool`` (or returns ``default`` if ``x`` is
falsy but not itself a boolean). Accepts various common string versions.
None, zero, blank string..."
91,create cookie,"def create_cookie(
key: str,
value: str='',
max_age: Optional[Union[int, timedelta]]=None,
expires: Optional[Union[int, float, datetime]]=None,
path: str='/',
domain: Optional[str]=None,
secure: bool=False,
httponly: bool=False,
) -> SimpleCookie:
cookie = SimpleCookie()
cookie[key] = value
cookie[key]['path'] = path
cookie[key]['httponly'] = httponly
cookie[key]['secure'] = secure
if isinstance(max_age, timedelta):
cookie[key]['max-age'] = f""{max_age.total_seconds():d}""
if isinstance(max_age, int):
cookie[key]['max-age'] = str(max_age)
if expires is not None and isinstance(expires, (int, float)):
cookie[key]['expires'] = format_date_time(int(expires))
elif expires is not None and isinstance(expires, datetime):
cookie[key]['expires'] = format_date_time(expires.replace(tzinfo=timezone.utc).timestamp())
if domain is not None:
cookie[key]['domain'] = domain
return cookie","Create a Cookie given the options set
The arguments are the standard cookie morsels and this is a
wrapper around the stdlib SimpleCookie code.
type: ignore
type: ignore"
92,get current ip address," def get_ip(self, address):
res = self.get_request('/ip_address/' + address)
return IPAddress(cloud_manager=self, **res['ip_address'])","Get an IPAddress object with the IP address (string) from the API.
e.g manager.get_ip('80.69.175.210')"
93,converting uint8 array to image,"def _convert_uint8(im):
if im.dtype != np.uint8:
im = np.uint8(im * 255)
return im",
94,how to read .csv file in an efficient way?,"def read_csv(csv_name, usecols=None):
csv_path = os.path.join(DATA_FOLDER, csv_name)
csv = pd.read_csv(csv_path, low_memory=False,
usecols=usecols, encoding=""utf-8"")
return csv",Returns a DataFrame from a .csv file stored in /data/raw/
95,export to excel,"def export_analytics_data_to_excel(data, output_file_name, result_info_key, identifier_keys):
workbook = create_excel_workbook(data, result_info_key, identifier_keys)
workbook.save(output_file_name)
print('Saved Excel file to {}'.format(output_file_name))","Creates an Excel file containing data returned by the Analytics API
Args:
data: Analytics API data as a list of dicts
output_file_name: File name for output Excel file (use .xlsx extension)."
96,unzipping large files,"def unzip_file(zip_fname):
print(""Unzipping {}"".format(zip_fname))
with zipfile.ZipFile(zip_fname) as zf:
zf.extractall()",Unzip the zip_fname in the current directory.
97,unzipping large files," def _unzip_file(self, src_path, dest_path, filename):
self.logger.info(""unzipping file..."")
unzip_path = os.path.join(dest_path, filename)
utils.ensure_directory_exists(unzip_path)
with zipfile.ZipFile(src_path, ""r"") as z:
z.extractall(unzip_path)
return True","unzips file located at src_path into destination_path
construct full path (including file name) for unzipping
extract data"
98,convert a utc time to epoch,"def datetime_to_epoch_seconds(value):
epoch = datetime.utcfromtimestamp(0).replace(tzinfo=timezone.utc)
return (value - epoch).total_seconds()",
99,write csv," def write_csv(self):
csv_path = self._path/'cleaned.csv'
with open(csv_path, 'w') as f:
csv_writer = csv.writer(f)
csv_writer.writerow(['name','label'])
for pair in self._csv_dict.items():
pair = [os.path.relpath(pair[0], self._path), pair[1]]
csv_writer.writerow(pair)
return csv_path",Get first element's file path so we write CSV to same directory as our data
100,parse binary file to custom class,"def parse_bin(path):
f = open(path, 'rb')
s = {}
output = []
methods = {
'GPS5': parse_gps,
'GPSU': parse_time,
'GPSF': parse_fix,
'GPSP': parse_precision,
'ACCL': parse_accl,
'GYRO': parse_gyro,
}
d = {'gps': []}
while True:
label = f.read(4)
if not label:
break
desc = f.read(4)
if '00' == binascii.hexlify(desc[0]):
continue
val_size = struct.unpack('>b', desc[1])[0]
num_values = struct.unpack('>h', desc[2:4])[0]
length = val_size * num_values
if label == 'DVID':
if len(d['gps']):
output.append(d)
d = {'gps': []}
for i in range(num_values):
data = f.read(val_size)
if label in methods:
methods[label](data, d, s)
if label == 'SCAL':
if 2 == val_size:
s[i] = struct.unpack('>h', data)[0]
elif 4 == val_size:
s[i] = struct.unpack('>i', data)[0]
else:
raise Exception('unknown scal size')
mod = length % 4
if mod != 0:
seek = 4 - mod
f.read(seek)
return output","the current Scale data to apply to next requester
handlers for various fourCC codes
up to date dictionary, iterate and fill then flush
eof
null length
print ""{} {} of size {} and type {}"".format(num_values, label,
val_size, desc[0])
first one is empty
reset
pack
discarded"
101,regex case insensitive,"def get_search_regex(query, ignore_case=True):
regex_text = [char for char in query if char != ' ']
regex_text = '.*'.join(regex_text)
regex = r'({0})'.format(regex_text)
if ignore_case:
pattern = re.compile(regex, re.IGNORECASE)
else:
pattern = re.compile(regex)
return pattern","Returns a compiled regex pattern to search for query letters in order.
Parameters
----------
query : str
String to search in another string (in order of character occurrence).
ignore_case : True
Optional value perform a case insensitive search (True by default).
Returns
-------
pattern : SRE_Pattern
Notes
-----
This function adds '.*' between the query characters and compiles the
resulting regular expression."
102,httpclient post json," def request(self, url,
json="""",
data="""",
username="""",
password="""",
headers=None,
timout=30):
raise NotImplementedError('request of HTTPClient should have been '
'overridden on initialization. '
'Otherwise, can be overridden to '
'supply your own post method')","This is overridden on module initialization. This function will make
an HTTP POST to a given url. Either json/data will be what is posted to
the end point. he HTTP request needs to be basicAuth when username and
password are provided. a headers dict maybe provided,
whatever the values are should be applied.
Args:
url (str): url to send the POST
json (dict, optional): Dict of the JSON to POST
data (dict, optional): Dict, presumed flat structure of
key/value of request to place as
www-form
username (str, optional): Username for basic auth. Must be
uncluded as part of password.
password (str, optional): Password for basic auth. Must be
included as part of username.
headers (dict, optional): Key/Value pairs of headers to include
Returns:
str: Raw request placed
str: Raw response received
int: HTTP status code, eg 200,404,401
dict: Key/Value pairs of the headers received.
:param timout:"
103,get executable path,"def find_windows_executable(bin_path, exe_name):
requested_path = get_windows_path(bin_path, exe_name)
if os.path.isfile(requested_path):
return requested_path
try:
pathext = os.environ[""PATHEXT""]
except KeyError:
pass
else:
for ext in pathext.split(os.pathsep):
path = get_windows_path(bin_path, exe_name + ext.strip().lower())
if os.path.isfile(path):
return path
return find_executable(exe_name)","Given an executable name, search the given location for an executable"
104,how to read .csv file in an efficient way?,"def read_csv(csv_name, usecols=None):
csv_path = os.path.join(DATA_FOLDER, csv_name)
csv = pd.read_csv(csv_path, low_memory=False,
usecols=usecols, encoding=""utf-8"")
return csv",Returns a DataFrame from a .csv file stored in /data/raw/
105,extract data from html content," def _ExtractDataFromShowHtml(self, html):
htmlLines = html.splitlines()
for count, line in enumerate(htmlLines):
if line.strip() == r'':
startLine = count+1
if line.strip() == r'':
endLine = count
try:
dataList = htmlLines[startLine:endLine]
dataString = '\n'.join(dataList)
return dataString.strip()
except:
raise Exception(""Show content not found - check EPGuides html formatting"")","Extracts csv show data from epguides html source.
Parameters
----------
html : string
Block of html text
Returns
----------
string
Show data extracted from html text in csv format."
106,heatmap from 3d coordinates," def get_figure(self, heatmap_kw=None, **kwargs):
if heatmap_kw is not None:
assert isinstance(heatmap_kw, dict)
if heatmap_kw is None:
heatmap_kw = {}
return self.get_heatmap(**heatmap_kw).get_figure(**kwargs)","Generate a plotly figure showing the matrix as a heatmap.
This is a shortcut for ``ExpMatrix.get_heatmap(...).get_figure(...)``.
See :func:`ExpHeatmap.get_figure` for keyword arguments.
Parameters
----------
heatmap_kw : dict or None
If not None, dictionary containing keyword arguments to be passed
to the `ExpHeatmap` constructor.
Returns
-------
`plotly.graph_objs.Figure`
The plotly figure."
107,confusion matrix," def from_existing(cls, confusion, *args, **kwargs):
df = []
for t, p in product(confusion.index.values, confusion.columns.values):
df += [[t, p]] * confusion[p][t]
if confusion.index.name is not None and confusion.columns.name is not None:
return Confusion(pd.DataFrame(df, columns=[confusion.index.name, confusion.columns.name]))
return Confusion(pd.DataFrame(df))","Creates a confusion matrix from a DataFrame that already contains confusion counts (but not meta stats)
>>> df = pd.DataFrame(np.matrix([[0,1,2,0,1,2,1,2,2,1],[0,1,2,1,2,0,0,1,2,0]]).T, columns=['True', 'Pred'])
>>> c = Confusion(df)
>>> c2 = pd.DataFrame(c)
>>> hasattr(c2, '_binary_sensitivity')
False
>>> c3 = Confusion.from_existing(c2)
>>> hasattr(c3, '_binary_sensitivity')
True
>>> (c3 == c).all().all()
True
>>> c3
Pred 0 1 2
True
0 1 1 0
1 2 1 1
2 1 1 2
Extremely brute-force to recreate data from a confusion matrix!"
108,confusion matrix," def confusion_matrix(self):
return plot.confusion_matrix(self.y_true, self.y_pred,
self.target_names, ax=_gen_ax())",Confusion matrix plot
109,get current process id,"def get_pid(PROCNAME):
for proc in psutil.process_iter():
if proc.name == PROCNAME:
return proc.pid",
110,copying a file to a path," def copy_file(self, from_path, to_path):
if not op.exists(op.dirname(to_path)):
self.make_directory(op.dirname(to_path))
shutil.copy(from_path, to_path)
logging.debug('File copied: {0}'.format(to_path))",Copy file.
111,how to get current date," def date(self):
if self._date:
return self._date
return datetime.datetime.now().strftime('%Y-%m-%d')","Getter/setter for the date member.
The setter can take a string or a :meth:`datetime.datetime` and will do the
appropriate transformation."
112,output to html file," def to_file(self, outputfile=DEFAULT_OUTPUTFILE):
if outputfile != NO_OUTPUTFILE:
if outputfile == DEFAULT_OUTPUTFILE:
outputfile = 'profile_' + str(hash(self)) + "".html""
with codecs.open(outputfile, 'w+b', encoding='utf8') as self.file:
self.file.write(templates.template('wrapper').render(content=self.html))","Write the report to a file.
By default a name is generated.
Parameters:
----------
outputfile : str
The name or the path of the file to generale including the extension (.html).
TODO: should be done in the template"
113,aes encryption,"def aes_ctr_encrypt(text, key, params):
iv = big_endian_to_int(decode_hex(params[""iv""]))
ctr = Counter.new(128, initial_value=iv, allow_wraparound=True)
mode = AES.MODE_CTR
encryptor = AES.new(key, mode, counter=ctr)
return encryptor.encrypt(text)",
114,encrypt aes ctr mode," def CTREnc(key, plaintext):
iv = os.urandom(AES.block_size)
cipher = _cipher(_aes(key), _ctrmode(iv), backend=_backend).encryptor()
return iv + cipher.update(plaintext) + cipher.finalize()",
115,binomial distribution," def sample(self, n=1):
p_vals = self._p_dist.rvs(size=n)[:, np.newaxis]
return np.random.binomial(self.n, p_vals)","numpy.random.binomial supports sampling using different p values,
whereas scipy does not."
116,replace in file," def _replace_in_file(self, dir_name, filename, args):
path = os.path.join(dir_name, filename)
with open(path, ""r"") as f:
s = f.read()
s = s.replace(""{%PROJECT_ID%}"", args.id)
s = s.replace(""{%PROJECT_ID_UPPER%}"", self._upper_slug)
s = s.replace(""{%PROJECT_ID_LOWER%}"", self._lower_slug)
s = s.replace(""{%PROJECT_NAME%}"", args.name)
s = s.replace(""{%PROJECT_GITHUB_USER%}"", args.github_user)
with open(path, ""w"") as f:
f.write(s)",
117,how to read .csv file in an efficient way?," def _read_csv(self, filename: str) -> pd.DataFrame:
path = self._pathmap.get(filename)
columns = self._config.nodes.get(filename, {}).get(""required_columns"", [])
if path is None or os.path.getsize(path) == 0:
return empty_df(columns)
with open(path, ""rb"") as f:
encoding = detect_encoding(f)
df = pd.read_csv(path, dtype=np.unicode, encoding=encoding, index_col=False)
df.rename(columns=lambda x: x.strip(), inplace=True)
if not df.empty:
for col in df.columns:
df[col] = df[col].str.strip()
return df","The file is missing or empty. Return an empty
DataFrame containing any required columns.
If the file isn't in the zip, return an empty DataFrame.
Strip leading/trailing whitespace from column names
Strip leading/trailing whitespace from column values"
118,encode url," def urlEncodeAndJoin(self, seq, sepr=','):
try:
from urllib.parse import quote_plus as encode
return sepr.join([encode(x, encoding=CHARSET_UTF8) for x in seq])
except ImportError:
from urllib import quote as encode
return sepr.join([i for i in map(lambda x: encode(x), seq)])","sepr.join(urlencode(seq))
Args:
seq: string list to be urlencoded
sepr: join seq with sepr
Returns:
str"
119,html entities replace,"def replace(html, replacements=None):
if not replacements:
return html
html = HTMLFragment(html)
for r in replacements:
r.replace(html)
return unicode(html)","Performs replacements on given HTML string.
no replacements"
120,export to excel," def to_excel(self, excel_writer, sheet_name='data',
iamc_index=False, **kwargs):
if not isinstance(excel_writer, pd.ExcelWriter):
close = True
excel_writer = pd.ExcelWriter(excel_writer)
self._to_file_format(iamc_index)\
.to_excel(excel_writer, sheet_name=sheet_name, index=False,
**kwargs)
if close:
excel_writer.close()","Write timeseries data to Excel format
Parameters
----------
excel_writer: string or ExcelWriter object
file path or existing ExcelWriter
sheet_name: string, default 'data'
name of sheet which will contain `IamDataFrame.timeseries()` data
iamc_index: bool, default False
if True, use `['model', 'scenario', 'region', 'variable', 'unit']`;
else, use all `data` columns"
121,heatmap from 3d coordinates," def draw(self, size=None, cmap=""jet""):
heatmaps_uint8 = self.to_uint8()
heatmaps_drawn = []
for c in sm.xrange(heatmaps_uint8.shape[2]):
heatmap_c = heatmaps_uint8[..., c:c+1]
if size is not None:
heatmap_c_rs = ia.imresize_single_image(heatmap_c, size, interpolation=""nearest"")
else:
heatmap_c_rs = heatmap_c
heatmap_c_rs = np.squeeze(heatmap_c_rs).astype(np.float32) / 255.0
if cmap is not None:
import matplotlib.pyplot as plt
cmap_func = plt.get_cmap(cmap)
heatmap_cmapped = cmap_func(heatmap_c_rs)
heatmap_cmapped = np.delete(heatmap_cmapped, 3, 2)
else:
heatmap_cmapped = np.tile(heatmap_c_rs[..., np.newaxis], (1, 1, 3))
heatmap_cmapped = np.clip(heatmap_cmapped * 255, 0, 255).astype(np.uint8)
heatmaps_drawn.append(heatmap_cmapped)
return heatmaps_drawn","Render the heatmaps as RGB images.
Parameters
----------
size : None or float or iterable of int or iterable of float, optional
Size of the rendered RGB image as ``(height, width)``.
See :func:`imgaug.imgaug.imresize_single_image` for details.
If set to None, no resizing is performed and the size of the heatmaps array is used.
cmap : str or None, optional
Color map of ``matplotlib`` to use in order to convert the heatmaps to RGB images.
If set to None, no color map will be used and the heatmaps will be converted
to simple intensity maps.
Returns
-------
heatmaps_drawn : list of (H,W,3) ndarray
Rendered heatmaps. One per heatmap array channel. Dtype is uint8.
c:c+1 here, because the additional axis is needed by imresize_single_image
import only when necessary (faster startup; optional dependency; less fragile -- see issue #225)"
122,how to get html of website," def get_html_source(self):
req = urllib.request.Request(self.url)
req.add_header(""user-agent"", random.choice(USER_AGENTS))
req_text = urllib.request.urlopen(req).read()
self.source = str(req_text)
self.soup = BeautifulSoup(self.source, ""html.parser"")
return self.source","Gets source page of url
:return: HTML source"
123,create cookie,"def create_cookie(
key: str,
value: str='',
max_age: Optional[Union[int, timedelta]]=None,
expires: Optional[Union[int, float, datetime]]=None,
path: str='/',
domain: Optional[str]=None,
secure: bool=False,
httponly: bool=False,
) -> SimpleCookie:
cookie = SimpleCookie()
cookie[key] = value
cookie[key]['path'] = path
cookie[key]['httponly'] = httponly
cookie[key]['secure'] = secure
if isinstance(max_age, timedelta):
cookie[key]['max-age'] = f""{max_age.total_seconds():d}""
if isinstance(max_age, int):
cookie[key]['max-age'] = str(max_age)
if expires is not None and isinstance(expires, (int, float)):
cookie[key]['expires'] = format_date_time(int(expires))
elif expires is not None and isinstance(expires, datetime):
cookie[key]['expires'] = format_date_time(expires.replace(tzinfo=timezone.utc).timestamp())
if domain is not None:
cookie[key]['domain'] = domain
return cookie","Create a Cookie given the options set
The arguments are the standard cookie morsels and this is a
wrapper around the stdlib SimpleCookie code.
type: ignore
type: ignore"
124,encode url," def trigger(self, identifier, force=True):
self.debug(identifier)
url = ""{base}/{identifier}"".format(
base=self.local_base_url,
identifier=identifier
)
param = {}
if force:
param['force'] = force
encode = urllib.urlencode(param)
if encode:
url += ""?""
url += encode
return self.core.update(url, {})",Trigger an upgrade task.
125,get inner html," def get_html(self, url):
r = self.get(url)
return self.html(r.text)",
126,copy to clipboard," def paste(self):
clipboard = QApplication.clipboard()
cliptext = ''
if clipboard.mimeData().hasText():
cliptext = to_text_string(clipboard.text())
if cliptext.strip():
self.import_from_string(cliptext, title=_(""Import from clipboard""))
else:
QMessageBox.warning(self, _( ""Empty clipboard""),
_(""Nothing to be imported from clipboard.""))
",Import text/data/code from clipboard
127,confusion matrix," def confusion_matrix(self):
confusion_matrix = self.pixel_classification_sum.astype(np.float)
confusion_matrix = np.divide(confusion_matrix.T, self.pixel_truth_sum.T).T
return confusion_matrix * 100.0",Returns the normalised confusion matrix
128,sending binary data over a serial connection," def send(self, msg):
slipDriver = sliplib.Driver()
slipData = slipDriver.send(msg)
res = self._serialPort.write(slipData)
return res","Encodes data to slip protocol and then sends over serial port
Uses the SlipLib module to convert the message data into SLIP format.
The message is then sent over the serial port opened with the instance
of the Faraday class used when invoking send().
Args:
msg (bytes): Bytes format message to send over serial port.
Returns:
int: Number of bytes transmitted over the serial port.
Create a sliplib Driver
Package data in slip format
Send data over serial port
Return number of bytes transmitted over serial port"
129,string similarity levenshtein,"def levenshtein(s1, s2, allow_substring=False):
len1, len2 = len(s1), len(s2)
lev = []
for i in range(len1 + 1):
lev.append([0] * (len2 + 1))
for i in range(len1 + 1):
lev[i][0] = i
for j in range(len2 + 1):
lev[0][j] = 0 if allow_substring else j
for i in range(len1):
for j in range(len2):
lev[i + 1][j + 1] = min(lev[i][j + 1] + 1, lev[i + 1][j] + 1, lev[i][j] + (s1[i] != s2[j]))
return min(lev[len1]) if allow_substring else lev[len1][len2]","Return the Levenshtein distance between two strings.
The Levenshtein distance (a.k.a ""edit difference"") is the number of characters that need to be substituted,
inserted or deleted to transform s1 into s2.
Setting the `allow_substring` parameter to True allows s1 to be a
substring of s2, so that, for example, ""hello"" and ""hello there"" would have a distance of zero.
:param string s1: The first string
:param string s2: The second string
:param bool allow_substring: Whether to allow s1 to be a substring of s2
:returns: Levenshtein distance.
:rtype int"
130,readonly array," def derive_readonly(self):
readonly = list(self.readonly)
for key, value in self.field_config.items():
if 'readonly' in value and value['readonly']:
readonly.append(key)
return readonly","Figures out what fields should be readonly. We iterate our field_config to find all
that have a readonly of true"
131,how to make the checkbox checked," def __call__(self, *arg):
if self.status:
self.status = 0
self.image.fill((255, 255, 255))
ptg.Button.set_position(self, self.position, self.midpoint,
self.surface)
else:
self.status = 1
if self.checktype == 'r':
self.draw_rect_check()
elif self.checktype == 'c':
self.draw_circle_check()
ptg.Button.set_position(self, self.position, self.midpoint,
self.surface)","If the checkbox was previously checked, set the status
to 0 and turn the whole box back to white.
Draw the checkbox on the external surface
If the checkbox was previously unchecked, set the status
to 1 and draw a check in the center of the checkbox.
Draw the new image on the external surface"
132,copy to clipboard,"def copy_to_clipboard(text):
if sys.platform == 'darwin':
os.system('echo ""{0}"" | pbcopy'.format(text))
return
try:
from Tkinter import Tk
except ImportError:
return
r = Tk()
r.withdraw()
r.clipboard_clear()
r.clipboard_append(text.encode('ascii'))
r.destroy()","reliable on mac
okay we'll try cross-platform way"
133,unique elements,"def unique_by_index(sequence):
uniques = []
for element in sequence:
if element not in uniques:
uniques.append(element)
return uniques","unique elements in `sequence` in the order in which they occur
Parameters
----------
sequence : iterable
Returns
-------
uniques : list
unique elements of sequence, ordered by the order in which the element
occurs in `sequence`"
134,finding time elapsed using a timer,"def timer(name):
startTime = time.time()
yield
elapsedTime = time.time() - startTime
print('[{}] finished in {} ms'.format(name, int(elapsedTime * 1000)))",
135,create cookie," def create_cookie(self, value, typ, cookie_name=None, ttl=-1, kill=False):
if kill:
ttl = -1
elif ttl < 0:
ttl = self.default_value['max_age']
if cookie_name is None:
cookie_name = self.default_value['name']
c_args = {}
srvdomain = self.default_value['domain']
if srvdomain and srvdomain not in ['localhost', '127.0.0.1',
'0.0.0.0']:
c_args['domain'] = srvdomain
srvpath = self.default_value['path']
if srvpath:
c_args['path'] = srvpath
timestamp = str(int(time.time()))
try:
cookie_payload = ""::"".join([value, timestamp, typ])
except TypeError:
cookie_payload = ""::"".join([value[0], timestamp, typ])
cookie = make_cookie(
cookie_name, cookie_payload, self.sign_key,
timestamp=timestamp, enc_key=self.enc_key, max_age=ttl,
sign_alg=self.sign_alg, **c_args)
return cookie",":param value: Part of the cookie payload
:param typ: Type of cookie
:param cookie_name:
:param ttl: Number of minutes before this cookie goes stale
:param kill: Whether the the cookie should expire on arrival
:return: A tuple to be added to headers
now
create cookie payload"
136,how to randomly pick a number,"def randbelow(num: int) -> int:
if not isinstance(num, int):
raise TypeError('number must be an integer')
if num <= 0:
raise ValueError('number must be greater than zero')
if num == 1:
return 0
nbits = num.bit_length()
randnum = random_randint(nbits)
while randnum >= num:
randnum = random_randint(nbits)
return randnum","Return a random int in the range [0,num).
Raises ValueError if num <= 0, and TypeError if it's not an integer.
>>> randbelow(16) #doctest:+SKIP
13
https://github.com/python/cpython/blob/3.6/Lib/random.py#L223
don't use (n-1) here because n can be 1
0 <= randnum < 2**nbits"
137,extracting data from a text file,"def extractdata(pattern, text=None, filepath=None):
y = []
if text is None:
textsource = open(filepath, 'r')
else:
textsource = text.splitlines()
for line in textsource:
match = scanf(pattern, line)
if match:
if len(y) == 0:
y = [[s] for s in match]
else:
for i, ydata in enumerate(y):
ydata.append(match[i])
if text is None:
textsource.close()
return y","Read through an entire file or body of text one line at a time. Parse each line that matches the supplied
pattern string and ignore the rest.
If *text* is supplied, it will be parsed according to the *pattern* string.
If *text* is not supplied, the file at *filepath* will be opened and parsed."
138,normal distribution," def normal_distribution(self, pos, sample):
curr_sigma_sq = self.sigma_sq(sample)
delta = self.trial_history[pos - 1] - self.f_comb(pos, sample)
return np.exp(np.square(delta) / (-2.0 * curr_sigma_sq)) / np.sqrt(2 * np.pi * np.sqrt(curr_sigma_sq))","returns the value of normal distribution, given the weight's sample and target position
Parameters
----------
pos: int
the epoch number of the position you want to predict
sample: list
sample is a (1 * NUM_OF_FUNCTIONS) matrix, representing{w1, w2, ... wk}
Returns
-------
float
the value of normal distribution"
139,matrix multiply," def multiply(self, matrix):
if not isinstance(matrix, DenseMatrix):
raise ValueError(""Only multiplication with DenseMatrix ""
""is supported."")
j_model = self._java_matrix_wrapper.call(""multiply"", matrix)
return RowMatrix(j_model)","Multiply this matrix by a local dense matrix on the right.
:param matrix: a local dense matrix whose number of rows must match the number of columns
of this matrix
:returns: :py:class:`RowMatrix`
>>> rm = RowMatrix(sc.parallelize([[0, 1], [2, 3]]))
>>> rm.multiply(DenseMatrix(2, 2, [0, 2, 1, 3])).rows.collect()
[DenseVector([2.0, 3.0]), DenseVector([6.0, 11.0])]"
140,convert decimal to hex," def hexadecimal(token):
token = ''.join([ c for c in token if c.isalnum() ])
if len(token) % 2 != 0:
raise ValueError(""Missing characters in hex data"")
data = ''
for i in compat.xrange(0, len(token), 2):
x = token[i:i+2]
d = int(x, 16)
s = struct.pack('"")
html_file.write(""Here are some graphs for you!
"")
for image in [lines_filename, bars_filename, histogram_filename]:
html_file.write(""{0}
